4277 lines · plain
1; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py2; RUN: llc < %s -verify-machineinstrs -mtriple=arm64-none-linux-gnu -fp-contract=fast | FileCheck %s --check-prefixes=CHECK,CHECK-SD3; RUN: llc < %s -verify-machineinstrs -mtriple=arm64-none-linux-gnu -fp-contract=fast -global-isel -global-isel-abort=2 2>&1 | FileCheck %s --check-prefixes=CHECK,CHECK-GI4 5; CHECK-GI: warning: Instruction selection used fallback path for test_vfms_lane_f326; CHECK-GI-NEXT: warning: Instruction selection used fallback path for test_vfmsq_lane_f327; CHECK-GI-NEXT: warning: Instruction selection used fallback path for test_vfms_laneq_f328; CHECK-GI-NEXT: warning: Instruction selection used fallback path for test_vfmsq_laneq_f329; CHECK-GI-NEXT: warning: Instruction selection used fallback path for test_vfmsq_lane_f6410; CHECK-GI-NEXT: warning: Instruction selection used fallback path for test_vfmsq_laneq_f6411; CHECK-GI-NEXT: warning: Instruction selection used fallback path for test_vfmsd_lane_f6412; CHECK-GI-NEXT: warning: Instruction selection used fallback path for test_vfmss_lane_f3213; CHECK-GI-NEXT: warning: Instruction selection used fallback path for test_vfmss_laneq_f3214; CHECK-GI-NEXT: warning: Instruction selection used fallback path for test_vfmsd_laneq_f6415; CHECK-GI-NEXT: warning: Instruction selection used fallback path for test_vfmsd_lane_f64_016; CHECK-GI-NEXT: warning: Instruction selection used fallback path for test_vfmss_lane_f32_017; CHECK-GI-NEXT: warning: Instruction selection used fallback path for test_vfmss_laneq_f32_018; CHECK-GI-NEXT: warning: Instruction selection used fallback path for test_vfmsd_laneq_f64_019; CHECK-GI-NEXT: warning: Instruction selection used fallback path for test_vqdmulh_lane_s16_intrinsic20; CHECK-GI-NEXT: warning: Instruction selection used fallback path for test_vqdmulh_laneq_s16_intrinsic_lo21; CHECK-GI-NEXT: warning: Instruction selection used fallback path for test_vqdmulh_laneq_s16_intrinsic_hi22; CHECK-GI-NEXT: warning: Instruction selection used fallback path for test_vqdmulhq_lane_s16_intrinsic23; CHECK-GI-NEXT: warning: Instruction selection used fallback path for test_vqdmulhq_laneq_s16_intrinsic_lo24; CHECK-GI-NEXT: warning: Instruction selection used fallback path for test_vqdmulhq_laneq_s16_intrinsic_hi25; CHECK-GI-NEXT: warning: Instruction selection used fallback path for test_vqdmulh_lane_s32_intrinsic26; CHECK-GI-NEXT: warning: Instruction selection used fallback path for test_vqdmulh_laneq_s32_intrinsic_lo27; CHECK-GI-NEXT: warning: Instruction selection used fallback path for test_vqdmulh_laneq_s32_intrinsic_hi28; CHECK-GI-NEXT: warning: Instruction selection used fallback path for test_vqdmulhq_lane_s32_intrinsic29; CHECK-GI-NEXT: warning: Instruction selection used fallback path for test_vqdmulhq_laneq_s32_intrinsic_lo30; CHECK-GI-NEXT: warning: Instruction selection used fallback path for test_vqdmulhq_laneq_s32_intrinsic_hi31; CHECK-GI-NEXT: warning: Instruction selection used fallback path for test_vqrdmulh_lane_s16_intrinsic32; CHECK-GI-NEXT: warning: Instruction selection used fallback path for test_vqrdmulh_laneq_s16_intrinsic_lo33; CHECK-GI-NEXT: warning: Instruction selection used fallback path for test_vqrdmulh_laneq_s16_intrinsic_hi34; CHECK-GI-NEXT: warning: Instruction selection used fallback path for test_vqrdmulhq_lane_s16_intrinsic35; CHECK-GI-NEXT: warning: Instruction selection used fallback path for test_vqrdmulhq_laneq_s16_intrinsic_lo36; CHECK-GI-NEXT: warning: Instruction selection used fallback path for test_vqrdmulhq_laneq_s16_intrinsic_hi37; CHECK-GI-NEXT: warning: Instruction selection used fallback path for test_vqrdmulh_lane_s32_intrinsic38; CHECK-GI-NEXT: warning: Instruction selection used fallback path for test_vqrdmulh_laneq_s32_intrinsic_lo39; CHECK-GI-NEXT: warning: Instruction selection used fallback path for test_vqrdmulh_laneq_s32_intrinsic_hi40; CHECK-GI-NEXT: warning: Instruction selection used fallback path for test_vqrdmulhq_lane_s32_intrinsic41; CHECK-GI-NEXT: warning: Instruction selection used fallback path for test_vqrdmulhq_laneq_s32_intrinsic_lo42; CHECK-GI-NEXT: warning: Instruction selection used fallback path for test_vqrdmulhq_laneq_s32_intrinsic_hi43; CHECK-GI-NEXT: warning: Instruction selection used fallback path for test_vfms_lane_f32_044; CHECK-GI-NEXT: warning: Instruction selection used fallback path for test_vfmsq_lane_f32_045; CHECK-GI-NEXT: warning: Instruction selection used fallback path for test_vfms_laneq_f32_046; CHECK-GI-NEXT: warning: Instruction selection used fallback path for test_vfmsq_laneq_f32_047; CHECK-GI-NEXT: warning: Instruction selection used fallback path for test_vfmsq_laneq_f64_048 49declare <2 x double> @llvm.aarch64.neon.fmulx.v2f64(<2 x double>, <2 x double>)50 51declare <4 x float> @llvm.aarch64.neon.fmulx.v4f32(<4 x float>, <4 x float>)52 53declare <2 x float> @llvm.aarch64.neon.fmulx.v2f32(<2 x float>, <2 x float>)54 55declare double @llvm.aarch64.neon.fmulx.f64(double, double)56 57declare <4 x i32> @llvm.aarch64.neon.sqrdmulh.v4i32(<4 x i32>, <4 x i32>)58declare <4 x i32> @llvm.aarch64.neon.sqrdmulh.lane.v4i32.v2i32(<4 x i32>, <2 x i32>, i32)59declare <4 x i32> @llvm.aarch64.neon.sqrdmulh.laneq.v4i32.v4i32(<4 x i32>, <4 x i32>, i32)60 61declare <2 x i32> @llvm.aarch64.neon.sqrdmulh.v2i32(<2 x i32>, <2 x i32>)62declare <2 x i32> @llvm.aarch64.neon.sqrdmulh.lane.v2i32.v2i32(<2 x i32>, <2 x i32>, i32)63declare <2 x i32> @llvm.aarch64.neon.sqrdmulh.laneq.v2i32.v4i32(<2 x i32>, <4 x i32>, i32)64 65declare <8 x i16> @llvm.aarch64.neon.sqrdmulh.v8i16(<8 x i16>, <8 x i16>)66declare <8 x i16> @llvm.aarch64.neon.sqrdmulh.lane.v8i16.v4i16(<8 x i16>, <4 x i16>, i32)67declare <8 x i16> @llvm.aarch64.neon.sqrdmulh.laneq.v8i16.v8i16(<8 x i16>, <8 x i16>, i32)68 69declare <4 x i16> @llvm.aarch64.neon.sqrdmulh.v4i16(<4 x i16>, <4 x i16>)70declare <4 x i16> @llvm.aarch64.neon.sqrdmulh.lane.v4i16.v4i16(<4 x i16>, <4 x i16>, i32)71declare <4 x i16> @llvm.aarch64.neon.sqrdmulh.laneq.v4i16.v8i16(<4 x i16>, <8 x i16>, i32)72 73declare <4 x i32> @llvm.aarch64.neon.sqdmulh.v4i32(<4 x i32>, <4 x i32>)74declare <4 x i32> @llvm.aarch64.neon.sqdmulh.lane.v4i32.v2i32(<4 x i32>, <2 x i32>, i32)75declare <4 x i32> @llvm.aarch64.neon.sqdmulh.laneq.v4i32.v4i32(<4 x i32>, <4 x i32>, i32)76 77declare <2 x i32> @llvm.aarch64.neon.sqdmulh.v2i32(<2 x i32>, <2 x i32>)78declare <2 x i32> @llvm.aarch64.neon.sqdmulh.lane.v2i32.v2i32(<2 x i32>, <2 x i32>, i32)79declare <2 x i32> @llvm.aarch64.neon.sqdmulh.laneq.v2i32.v4i32(<2 x i32>, <4 x i32>, i32)80 81declare <8 x i16> @llvm.aarch64.neon.sqdmulh.v8i16(<8 x i16>, <8 x i16>)82declare <8 x i16> @llvm.aarch64.neon.sqdmulh.lane.v8i16.v4i16(<8 x i16>, <4 x i16>, i32)83declare <8 x i16> @llvm.aarch64.neon.sqdmulh.laneq.v8i16.v8i16(<8 x i16>, <8 x i16>, i32)84 85declare <4 x i16> @llvm.aarch64.neon.sqdmulh.v4i16(<4 x i16>, <4 x i16>)86declare <4 x i16> @llvm.aarch64.neon.sqdmulh.lane.v4i16.v4i16(<4 x i16>, <4 x i16>, i32)87declare <4 x i16> @llvm.aarch64.neon.sqdmulh.laneq.v4i16.v8i16(<4 x i16>, <8 x i16>, i32)88 89declare <2 x i64> @llvm.aarch64.neon.sqdmull.v2i64(<2 x i32>, <2 x i32>)90 91declare <4 x i32> @llvm.aarch64.neon.sqdmull.v4i32(<4 x i16>, <4 x i16>)92 93declare <2 x i64> @llvm.aarch64.neon.sqsub.v2i64(<2 x i64>, <2 x i64>)94 95declare <4 x i32> @llvm.aarch64.neon.sqsub.v4i32(<4 x i32>, <4 x i32>)96 97declare <2 x i64> @llvm.aarch64.neon.sqadd.v2i64(<2 x i64>, <2 x i64>)98 99declare <4 x i32> @llvm.aarch64.neon.sqadd.v4i32(<4 x i32>, <4 x i32>)100 101declare <2 x i64> @llvm.aarch64.neon.umull.v2i64(<2 x i32>, <2 x i32>)102 103declare <4 x i32> @llvm.aarch64.neon.umull.v4i32(<4 x i16>, <4 x i16>)104 105declare <2 x i64> @llvm.aarch64.neon.smull.v2i64(<2 x i32>, <2 x i32>)106 107declare <4 x i32> @llvm.aarch64.neon.smull.v4i32(<4 x i16>, <4 x i16>)108 109define <4 x i16> @test_vmla_lane_s16(<4 x i16> %a, <4 x i16> %b, <4 x i16> %v) {110; CHECK-LABEL: test_vmla_lane_s16:111; CHECK: // %bb.0: // %entry112; CHECK-NEXT: // kill: def $d2 killed $d2 def $q2113; CHECK-NEXT: mla v0.4h, v1.4h, v2.h[3]114; CHECK-NEXT: ret115entry:116 %shuffle = shufflevector <4 x i16> %v, <4 x i16> undef, <4 x i32> <i32 3, i32 3, i32 3, i32 3>117 %mul = mul <4 x i16> %shuffle, %b118 %add = add <4 x i16> %mul, %a119 ret <4 x i16> %add120}121 122define <8 x i16> @test_vmlaq_lane_s16(<8 x i16> %a, <8 x i16> %b, <4 x i16> %v) {123; CHECK-LABEL: test_vmlaq_lane_s16:124; CHECK: // %bb.0: // %entry125; CHECK-NEXT: // kill: def $d2 killed $d2 def $q2126; CHECK-NEXT: mla v0.8h, v1.8h, v2.h[3]127; CHECK-NEXT: ret128entry:129 %shuffle = shufflevector <4 x i16> %v, <4 x i16> undef, <8 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>130 %mul = mul <8 x i16> %shuffle, %b131 %add = add <8 x i16> %mul, %a132 ret <8 x i16> %add133}134 135define <2 x i32> @test_vmla_lane_s32(<2 x i32> %a, <2 x i32> %b, <2 x i32> %v) {136; CHECK-LABEL: test_vmla_lane_s32:137; CHECK: // %bb.0: // %entry138; CHECK-NEXT: // kill: def $d2 killed $d2 def $q2139; CHECK-NEXT: mla v0.2s, v1.2s, v2.s[1]140; CHECK-NEXT: ret141entry:142 %shuffle = shufflevector <2 x i32> %v, <2 x i32> undef, <2 x i32> <i32 1, i32 1>143 %mul = mul <2 x i32> %shuffle, %b144 %add = add <2 x i32> %mul, %a145 ret <2 x i32> %add146}147 148define <4 x i32> @test_vmlaq_lane_s32(<4 x i32> %a, <4 x i32> %b, <2 x i32> %v) {149; CHECK-LABEL: test_vmlaq_lane_s32:150; CHECK: // %bb.0: // %entry151; CHECK-NEXT: // kill: def $d2 killed $d2 def $q2152; CHECK-NEXT: mla v0.4s, v1.4s, v2.s[1]153; CHECK-NEXT: ret154entry:155 %shuffle = shufflevector <2 x i32> %v, <2 x i32> undef, <4 x i32> <i32 1, i32 1, i32 1, i32 1>156 %mul = mul <4 x i32> %shuffle, %b157 %add = add <4 x i32> %mul, %a158 ret <4 x i32> %add159}160 161define <4 x i16> @test_vmla_laneq_s16(<4 x i16> %a, <4 x i16> %b, <8 x i16> %v) {162; CHECK-LABEL: test_vmla_laneq_s16:163; CHECK: // %bb.0: // %entry164; CHECK-NEXT: mla v0.4h, v1.4h, v2.h[7]165; CHECK-NEXT: ret166entry:167 %shuffle = shufflevector <8 x i16> %v, <8 x i16> undef, <4 x i32> <i32 7, i32 7, i32 7, i32 7>168 %mul = mul <4 x i16> %shuffle, %b169 %add = add <4 x i16> %mul, %a170 ret <4 x i16> %add171}172 173define <8 x i16> @test_vmlaq_laneq_s16(<8 x i16> %a, <8 x i16> %b, <8 x i16> %v) {174; CHECK-LABEL: test_vmlaq_laneq_s16:175; CHECK: // %bb.0: // %entry176; CHECK-NEXT: mla v0.8h, v1.8h, v2.h[7]177; CHECK-NEXT: ret178entry:179 %shuffle = shufflevector <8 x i16> %v, <8 x i16> undef, <8 x i32> <i32 7, i32 7, i32 7, i32 7, i32 7, i32 7, i32 7, i32 7>180 %mul = mul <8 x i16> %shuffle, %b181 %add = add <8 x i16> %mul, %a182 ret <8 x i16> %add183}184 185define <2 x i32> @test_vmla_laneq_s32(<2 x i32> %a, <2 x i32> %b, <4 x i32> %v) {186; CHECK-LABEL: test_vmla_laneq_s32:187; CHECK: // %bb.0: // %entry188; CHECK-NEXT: mla v0.2s, v1.2s, v2.s[3]189; CHECK-NEXT: ret190entry:191 %shuffle = shufflevector <4 x i32> %v, <4 x i32> undef, <2 x i32> <i32 3, i32 3>192 %mul = mul <2 x i32> %shuffle, %b193 %add = add <2 x i32> %mul, %a194 ret <2 x i32> %add195}196 197define <4 x i32> @test_vmlaq_laneq_s32(<4 x i32> %a, <4 x i32> %b, <4 x i32> %v) {198; CHECK-LABEL: test_vmlaq_laneq_s32:199; CHECK: // %bb.0: // %entry200; CHECK-NEXT: mla v0.4s, v1.4s, v2.s[3]201; CHECK-NEXT: ret202entry:203 %shuffle = shufflevector <4 x i32> %v, <4 x i32> undef, <4 x i32> <i32 3, i32 3, i32 3, i32 3>204 %mul = mul <4 x i32> %shuffle, %b205 %add = add <4 x i32> %mul, %a206 ret <4 x i32> %add207}208 209define <4 x i16> @test_vmls_lane_s16(<4 x i16> %a, <4 x i16> %b, <4 x i16> %v) {210; CHECK-LABEL: test_vmls_lane_s16:211; CHECK: // %bb.0: // %entry212; CHECK-NEXT: // kill: def $d2 killed $d2 def $q2213; CHECK-NEXT: mls v0.4h, v1.4h, v2.h[3]214; CHECK-NEXT: ret215entry:216 %shuffle = shufflevector <4 x i16> %v, <4 x i16> undef, <4 x i32> <i32 3, i32 3, i32 3, i32 3>217 %mul = mul <4 x i16> %shuffle, %b218 %sub = sub <4 x i16> %a, %mul219 ret <4 x i16> %sub220}221 222define <8 x i16> @test_vmlsq_lane_s16(<8 x i16> %a, <8 x i16> %b, <4 x i16> %v) {223; CHECK-LABEL: test_vmlsq_lane_s16:224; CHECK: // %bb.0: // %entry225; CHECK-NEXT: // kill: def $d2 killed $d2 def $q2226; CHECK-NEXT: mls v0.8h, v1.8h, v2.h[3]227; CHECK-NEXT: ret228entry:229 %shuffle = shufflevector <4 x i16> %v, <4 x i16> undef, <8 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>230 %mul = mul <8 x i16> %shuffle, %b231 %sub = sub <8 x i16> %a, %mul232 ret <8 x i16> %sub233}234 235define <2 x i32> @test_vmls_lane_s32(<2 x i32> %a, <2 x i32> %b, <2 x i32> %v) {236; CHECK-LABEL: test_vmls_lane_s32:237; CHECK: // %bb.0: // %entry238; CHECK-NEXT: // kill: def $d2 killed $d2 def $q2239; CHECK-NEXT: mls v0.2s, v1.2s, v2.s[1]240; CHECK-NEXT: ret241entry:242 %shuffle = shufflevector <2 x i32> %v, <2 x i32> undef, <2 x i32> <i32 1, i32 1>243 %mul = mul <2 x i32> %shuffle, %b244 %sub = sub <2 x i32> %a, %mul245 ret <2 x i32> %sub246}247 248define <4 x i32> @test_vmlsq_lane_s32(<4 x i32> %a, <4 x i32> %b, <2 x i32> %v) {249; CHECK-LABEL: test_vmlsq_lane_s32:250; CHECK: // %bb.0: // %entry251; CHECK-NEXT: // kill: def $d2 killed $d2 def $q2252; CHECK-NEXT: mls v0.4s, v1.4s, v2.s[1]253; CHECK-NEXT: ret254entry:255 %shuffle = shufflevector <2 x i32> %v, <2 x i32> undef, <4 x i32> <i32 1, i32 1, i32 1, i32 1>256 %mul = mul <4 x i32> %shuffle, %b257 %sub = sub <4 x i32> %a, %mul258 ret <4 x i32> %sub259}260 261define <4 x i16> @test_vmls_laneq_s16(<4 x i16> %a, <4 x i16> %b, <8 x i16> %v) {262; CHECK-LABEL: test_vmls_laneq_s16:263; CHECK: // %bb.0: // %entry264; CHECK-NEXT: mls v0.4h, v1.4h, v2.h[7]265; CHECK-NEXT: ret266entry:267 %shuffle = shufflevector <8 x i16> %v, <8 x i16> undef, <4 x i32> <i32 7, i32 7, i32 7, i32 7>268 %mul = mul <4 x i16> %shuffle, %b269 %sub = sub <4 x i16> %a, %mul270 ret <4 x i16> %sub271}272 273define <8 x i16> @test_vmlsq_laneq_s16(<8 x i16> %a, <8 x i16> %b, <8 x i16> %v) {274; CHECK-LABEL: test_vmlsq_laneq_s16:275; CHECK: // %bb.0: // %entry276; CHECK-NEXT: mls v0.8h, v1.8h, v2.h[7]277; CHECK-NEXT: ret278entry:279 %shuffle = shufflevector <8 x i16> %v, <8 x i16> undef, <8 x i32> <i32 7, i32 7, i32 7, i32 7, i32 7, i32 7, i32 7, i32 7>280 %mul = mul <8 x i16> %shuffle, %b281 %sub = sub <8 x i16> %a, %mul282 ret <8 x i16> %sub283}284 285define <2 x i32> @test_vmls_laneq_s32(<2 x i32> %a, <2 x i32> %b, <4 x i32> %v) {286; CHECK-LABEL: test_vmls_laneq_s32:287; CHECK: // %bb.0: // %entry288; CHECK-NEXT: mls v0.2s, v1.2s, v2.s[3]289; CHECK-NEXT: ret290entry:291 %shuffle = shufflevector <4 x i32> %v, <4 x i32> undef, <2 x i32> <i32 3, i32 3>292 %mul = mul <2 x i32> %shuffle, %b293 %sub = sub <2 x i32> %a, %mul294 ret <2 x i32> %sub295}296 297define <4 x i32> @test_vmlsq_laneq_s32(<4 x i32> %a, <4 x i32> %b, <4 x i32> %v) {298; CHECK-LABEL: test_vmlsq_laneq_s32:299; CHECK: // %bb.0: // %entry300; CHECK-NEXT: mls v0.4s, v1.4s, v2.s[3]301; CHECK-NEXT: ret302entry:303 %shuffle = shufflevector <4 x i32> %v, <4 x i32> undef, <4 x i32> <i32 3, i32 3, i32 3, i32 3>304 %mul = mul <4 x i32> %shuffle, %b305 %sub = sub <4 x i32> %a, %mul306 ret <4 x i32> %sub307}308 309define <4 x i16> @test_vmul_lane_s16(<4 x i16> %a, <4 x i16> %v) {310; CHECK-LABEL: test_vmul_lane_s16:311; CHECK: // %bb.0: // %entry312; CHECK-NEXT: // kill: def $d1 killed $d1 def $q1313; CHECK-NEXT: mul v0.4h, v0.4h, v1.h[3]314; CHECK-NEXT: ret315entry:316 %shuffle = shufflevector <4 x i16> %v, <4 x i16> undef, <4 x i32> <i32 3, i32 3, i32 3, i32 3>317 %mul = mul <4 x i16> %shuffle, %a318 ret <4 x i16> %mul319}320 321define <8 x i16> @test_vmulq_lane_s16(<8 x i16> %a, <4 x i16> %v) {322; CHECK-LABEL: test_vmulq_lane_s16:323; CHECK: // %bb.0: // %entry324; CHECK-NEXT: // kill: def $d1 killed $d1 def $q1325; CHECK-NEXT: mul v0.8h, v0.8h, v1.h[3]326; CHECK-NEXT: ret327entry:328 %shuffle = shufflevector <4 x i16> %v, <4 x i16> undef, <8 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>329 %mul = mul <8 x i16> %shuffle, %a330 ret <8 x i16> %mul331}332 333define <2 x i32> @test_vmul_lane_s32(<2 x i32> %a, <2 x i32> %v) {334; CHECK-LABEL: test_vmul_lane_s32:335; CHECK: // %bb.0: // %entry336; CHECK-NEXT: // kill: def $d1 killed $d1 def $q1337; CHECK-NEXT: mul v0.2s, v0.2s, v1.s[1]338; CHECK-NEXT: ret339entry:340 %shuffle = shufflevector <2 x i32> %v, <2 x i32> undef, <2 x i32> <i32 1, i32 1>341 %mul = mul <2 x i32> %shuffle, %a342 ret <2 x i32> %mul343}344 345define <4 x i32> @test_vmulq_lane_s32(<4 x i32> %a, <2 x i32> %v) {346; CHECK-LABEL: test_vmulq_lane_s32:347; CHECK: // %bb.0: // %entry348; CHECK-NEXT: // kill: def $d1 killed $d1 def $q1349; CHECK-NEXT: mul v0.4s, v0.4s, v1.s[1]350; CHECK-NEXT: ret351entry:352 %shuffle = shufflevector <2 x i32> %v, <2 x i32> undef, <4 x i32> <i32 1, i32 1, i32 1, i32 1>353 %mul = mul <4 x i32> %shuffle, %a354 ret <4 x i32> %mul355}356 357define <4 x i16> @test_vmul_lane_u16(<4 x i16> %a, <4 x i16> %v) {358; CHECK-LABEL: test_vmul_lane_u16:359; CHECK: // %bb.0: // %entry360; CHECK-NEXT: // kill: def $d1 killed $d1 def $q1361; CHECK-NEXT: mul v0.4h, v0.4h, v1.h[3]362; CHECK-NEXT: ret363entry:364 %shuffle = shufflevector <4 x i16> %v, <4 x i16> undef, <4 x i32> <i32 3, i32 3, i32 3, i32 3>365 %mul = mul <4 x i16> %shuffle, %a366 ret <4 x i16> %mul367}368 369define <8 x i16> @test_vmulq_lane_u16(<8 x i16> %a, <4 x i16> %v) {370; CHECK-LABEL: test_vmulq_lane_u16:371; CHECK: // %bb.0: // %entry372; CHECK-NEXT: // kill: def $d1 killed $d1 def $q1373; CHECK-NEXT: mul v0.8h, v0.8h, v1.h[3]374; CHECK-NEXT: ret375entry:376 %shuffle = shufflevector <4 x i16> %v, <4 x i16> undef, <8 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>377 %mul = mul <8 x i16> %shuffle, %a378 ret <8 x i16> %mul379}380 381define <2 x i32> @test_vmul_lane_u32(<2 x i32> %a, <2 x i32> %v) {382; CHECK-LABEL: test_vmul_lane_u32:383; CHECK: // %bb.0: // %entry384; CHECK-NEXT: // kill: def $d1 killed $d1 def $q1385; CHECK-NEXT: mul v0.2s, v0.2s, v1.s[1]386; CHECK-NEXT: ret387entry:388 %shuffle = shufflevector <2 x i32> %v, <2 x i32> undef, <2 x i32> <i32 1, i32 1>389 %mul = mul <2 x i32> %shuffle, %a390 ret <2 x i32> %mul391}392 393define <4 x i32> @test_vmulq_lane_u32(<4 x i32> %a, <2 x i32> %v) {394; CHECK-LABEL: test_vmulq_lane_u32:395; CHECK: // %bb.0: // %entry396; CHECK-NEXT: // kill: def $d1 killed $d1 def $q1397; CHECK-NEXT: mul v0.4s, v0.4s, v1.s[1]398; CHECK-NEXT: ret399entry:400 %shuffle = shufflevector <2 x i32> %v, <2 x i32> undef, <4 x i32> <i32 1, i32 1, i32 1, i32 1>401 %mul = mul <4 x i32> %shuffle, %a402 ret <4 x i32> %mul403}404 405define <4 x i16> @test_vmul_laneq_s16(<4 x i16> %a, <8 x i16> %v) {406; CHECK-LABEL: test_vmul_laneq_s16:407; CHECK: // %bb.0: // %entry408; CHECK-NEXT: mul v0.4h, v0.4h, v1.h[7]409; CHECK-NEXT: ret410entry:411 %shuffle = shufflevector <8 x i16> %v, <8 x i16> undef, <4 x i32> <i32 7, i32 7, i32 7, i32 7>412 %mul = mul <4 x i16> %shuffle, %a413 ret <4 x i16> %mul414}415 416define <8 x i16> @test_vmulq_laneq_s16(<8 x i16> %a, <8 x i16> %v) {417; CHECK-LABEL: test_vmulq_laneq_s16:418; CHECK: // %bb.0: // %entry419; CHECK-NEXT: mul v0.8h, v0.8h, v1.h[7]420; CHECK-NEXT: ret421entry:422 %shuffle = shufflevector <8 x i16> %v, <8 x i16> undef, <8 x i32> <i32 7, i32 7, i32 7, i32 7, i32 7, i32 7, i32 7, i32 7>423 %mul = mul <8 x i16> %shuffle, %a424 ret <8 x i16> %mul425}426 427define <2 x i32> @test_vmul_laneq_s32(<2 x i32> %a, <4 x i32> %v) {428; CHECK-LABEL: test_vmul_laneq_s32:429; CHECK: // %bb.0: // %entry430; CHECK-NEXT: mul v0.2s, v0.2s, v1.s[3]431; CHECK-NEXT: ret432entry:433 %shuffle = shufflevector <4 x i32> %v, <4 x i32> undef, <2 x i32> <i32 3, i32 3>434 %mul = mul <2 x i32> %shuffle, %a435 ret <2 x i32> %mul436}437 438define <4 x i32> @test_vmulq_laneq_s32(<4 x i32> %a, <4 x i32> %v) {439; CHECK-LABEL: test_vmulq_laneq_s32:440; CHECK: // %bb.0: // %entry441; CHECK-NEXT: mul v0.4s, v0.4s, v1.s[3]442; CHECK-NEXT: ret443entry:444 %shuffle = shufflevector <4 x i32> %v, <4 x i32> undef, <4 x i32> <i32 3, i32 3, i32 3, i32 3>445 %mul = mul <4 x i32> %shuffle, %a446 ret <4 x i32> %mul447}448 449define <4 x i16> @test_vmul_laneq_u16(<4 x i16> %a, <8 x i16> %v) {450; CHECK-LABEL: test_vmul_laneq_u16:451; CHECK: // %bb.0: // %entry452; CHECK-NEXT: mul v0.4h, v0.4h, v1.h[7]453; CHECK-NEXT: ret454entry:455 %shuffle = shufflevector <8 x i16> %v, <8 x i16> undef, <4 x i32> <i32 7, i32 7, i32 7, i32 7>456 %mul = mul <4 x i16> %shuffle, %a457 ret <4 x i16> %mul458}459 460define <8 x i16> @test_vmulq_laneq_u16(<8 x i16> %a, <8 x i16> %v) {461; CHECK-LABEL: test_vmulq_laneq_u16:462; CHECK: // %bb.0: // %entry463; CHECK-NEXT: mul v0.8h, v0.8h, v1.h[7]464; CHECK-NEXT: ret465entry:466 %shuffle = shufflevector <8 x i16> %v, <8 x i16> undef, <8 x i32> <i32 7, i32 7, i32 7, i32 7, i32 7, i32 7, i32 7, i32 7>467 %mul = mul <8 x i16> %shuffle, %a468 ret <8 x i16> %mul469}470 471define <2 x i32> @test_vmul_laneq_u32(<2 x i32> %a, <4 x i32> %v) {472; CHECK-LABEL: test_vmul_laneq_u32:473; CHECK: // %bb.0: // %entry474; CHECK-NEXT: mul v0.2s, v0.2s, v1.s[3]475; CHECK-NEXT: ret476entry:477 %shuffle = shufflevector <4 x i32> %v, <4 x i32> undef, <2 x i32> <i32 3, i32 3>478 %mul = mul <2 x i32> %shuffle, %a479 ret <2 x i32> %mul480}481 482define <4 x i32> @test_vmulq_laneq_u32(<4 x i32> %a, <4 x i32> %v) {483; CHECK-LABEL: test_vmulq_laneq_u32:484; CHECK: // %bb.0: // %entry485; CHECK-NEXT: mul v0.4s, v0.4s, v1.s[3]486; CHECK-NEXT: ret487entry:488 %shuffle = shufflevector <4 x i32> %v, <4 x i32> undef, <4 x i32> <i32 3, i32 3, i32 3, i32 3>489 %mul = mul <4 x i32> %shuffle, %a490 ret <4 x i32> %mul491}492 493define <2 x float> @test_vfma_lane_f32(<2 x float> %a, <2 x float> %b, <2 x float> %v) {494; CHECK-LABEL: test_vfma_lane_f32:495; CHECK: // %bb.0: // %entry496; CHECK-NEXT: // kill: def $d2 killed $d2 def $q2497; CHECK-NEXT: fmla v0.2s, v1.2s, v2.s[1]498; CHECK-NEXT: ret499entry:500 %lane = shufflevector <2 x float> %v, <2 x float> undef, <2 x i32> <i32 1, i32 1>501 %0 = tail call <2 x float> @llvm.fma.v2f32(<2 x float> %lane, <2 x float> %b, <2 x float> %a)502 ret <2 x float> %0503}504 505declare <2 x float> @llvm.fma.v2f32(<2 x float>, <2 x float>, <2 x float>)506 507define <4 x float> @test_vfmaq_lane_f32(<4 x float> %a, <4 x float> %b, <2 x float> %v) {508; CHECK-LABEL: test_vfmaq_lane_f32:509; CHECK: // %bb.0: // %entry510; CHECK-NEXT: // kill: def $d2 killed $d2 def $q2511; CHECK-NEXT: fmla v0.4s, v1.4s, v2.s[1]512; CHECK-NEXT: ret513entry:514 %lane = shufflevector <2 x float> %v, <2 x float> undef, <4 x i32> <i32 1, i32 1, i32 1, i32 1>515 %0 = tail call <4 x float> @llvm.fma.v4f32(<4 x float> %lane, <4 x float> %b, <4 x float> %a)516 ret <4 x float> %0517}518 519declare <4 x float> @llvm.fma.v4f32(<4 x float>, <4 x float>, <4 x float>)520 521define <2 x float> @test_vfma_laneq_f32(<2 x float> %a, <2 x float> %b, <4 x float> %v) {522; CHECK-LABEL: test_vfma_laneq_f32:523; CHECK: // %bb.0: // %entry524; CHECK-NEXT: fmla v0.2s, v1.2s, v2.s[3]525; CHECK-NEXT: ret526entry:527 %lane = shufflevector <4 x float> %v, <4 x float> undef, <2 x i32> <i32 3, i32 3>528 %0 = tail call <2 x float> @llvm.fma.v2f32(<2 x float> %lane, <2 x float> %b, <2 x float> %a)529 ret <2 x float> %0530}531 532define <4 x float> @test_vfmaq_laneq_f32(<4 x float> %a, <4 x float> %b, <4 x float> %v) {533; CHECK-LABEL: test_vfmaq_laneq_f32:534; CHECK: // %bb.0: // %entry535; CHECK-NEXT: fmla v0.4s, v1.4s, v2.s[3]536; CHECK-NEXT: ret537entry:538 %lane = shufflevector <4 x float> %v, <4 x float> undef, <4 x i32> <i32 3, i32 3, i32 3, i32 3>539 %0 = tail call <4 x float> @llvm.fma.v4f32(<4 x float> %lane, <4 x float> %b, <4 x float> %a)540 ret <4 x float> %0541}542 543define <2 x float> @test_vfms_lane_f32(<2 x float> %a, <2 x float> %b, <2 x float> %v) {544; CHECK-LABEL: test_vfms_lane_f32:545; CHECK: // %bb.0: // %entry546; CHECK-NEXT: // kill: def $d2 killed $d2 def $q2547; CHECK-NEXT: fmls v0.2s, v1.2s, v2.s[1]548; CHECK-NEXT: ret549entry:550 %sub = fsub <2 x float> <float -0.000000e+00, float -0.000000e+00>, %v551 %lane = shufflevector <2 x float> %sub, <2 x float> undef, <2 x i32> <i32 1, i32 1>552 %0 = tail call <2 x float> @llvm.fma.v2f32(<2 x float> %lane, <2 x float> %b, <2 x float> %a)553 ret <2 x float> %0554}555 556define <4 x float> @test_vfmsq_lane_f32(<4 x float> %a, <4 x float> %b, <2 x float> %v) {557; CHECK-LABEL: test_vfmsq_lane_f32:558; CHECK: // %bb.0: // %entry559; CHECK-NEXT: // kill: def $d2 killed $d2 def $q2560; CHECK-NEXT: fmls v0.4s, v1.4s, v2.s[1]561; CHECK-NEXT: ret562entry:563 %sub = fsub <2 x float> <float -0.000000e+00, float -0.000000e+00>, %v564 %lane = shufflevector <2 x float> %sub, <2 x float> undef, <4 x i32> <i32 1, i32 1, i32 1, i32 1>565 %0 = tail call <4 x float> @llvm.fma.v4f32(<4 x float> %lane, <4 x float> %b, <4 x float> %a)566 ret <4 x float> %0567}568 569define <2 x float> @test_vfms_laneq_f32(<2 x float> %a, <2 x float> %b, <4 x float> %v) {570; CHECK-LABEL: test_vfms_laneq_f32:571; CHECK: // %bb.0: // %entry572; CHECK-NEXT: fmls v0.2s, v1.2s, v2.s[3]573; CHECK-NEXT: ret574entry:575 %sub = fsub <4 x float> <float -0.000000e+00, float -0.000000e+00, float -0.000000e+00, float -0.000000e+00>, %v576 %lane = shufflevector <4 x float> %sub, <4 x float> undef, <2 x i32> <i32 3, i32 3>577 %0 = tail call <2 x float> @llvm.fma.v2f32(<2 x float> %lane, <2 x float> %b, <2 x float> %a)578 ret <2 x float> %0579}580 581define <4 x float> @test_vfmsq_laneq_f32(<4 x float> %a, <4 x float> %b, <4 x float> %v) {582; CHECK-LABEL: test_vfmsq_laneq_f32:583; CHECK: // %bb.0: // %entry584; CHECK-NEXT: fmls v0.4s, v1.4s, v2.s[3]585; CHECK-NEXT: ret586entry:587 %sub = fsub <4 x float> <float -0.000000e+00, float -0.000000e+00, float -0.000000e+00, float -0.000000e+00>, %v588 %lane = shufflevector <4 x float> %sub, <4 x float> undef, <4 x i32> <i32 3, i32 3, i32 3, i32 3>589 %0 = tail call <4 x float> @llvm.fma.v4f32(<4 x float> %lane, <4 x float> %b, <4 x float> %a)590 ret <4 x float> %0591}592 593define <2 x double> @test_vfmaq_lane_f64(<2 x double> %a, <2 x double> %b, <1 x double> %v) {594; CHECK-LABEL: test_vfmaq_lane_f64:595; CHECK: // %bb.0: // %entry596; CHECK-NEXT: // kill: def $d2 killed $d2 def $q2597; CHECK-NEXT: fmla v0.2d, v1.2d, v2.d[0]598; CHECK-NEXT: ret599entry:600 %lane = shufflevector <1 x double> %v, <1 x double> undef, <2 x i32> zeroinitializer601 %0 = tail call <2 x double> @llvm.fma.v2f64(<2 x double> %lane, <2 x double> %b, <2 x double> %a)602 ret <2 x double> %0603}604 605declare <2 x double> @llvm.fma.v2f64(<2 x double>, <2 x double>, <2 x double>)606 607define <2 x double> @test_vfmaq_laneq_f64(<2 x double> %a, <2 x double> %b, <2 x double> %v) {608; CHECK-LABEL: test_vfmaq_laneq_f64:609; CHECK: // %bb.0: // %entry610; CHECK-NEXT: fmla v0.2d, v1.2d, v2.d[1]611; CHECK-NEXT: ret612entry:613 %lane = shufflevector <2 x double> %v, <2 x double> undef, <2 x i32> <i32 1, i32 1>614 %0 = tail call <2 x double> @llvm.fma.v2f64(<2 x double> %lane, <2 x double> %b, <2 x double> %a)615 ret <2 x double> %0616}617 618define <2 x double> @test_vfmsq_lane_f64(<2 x double> %a, <2 x double> %b, <1 x double> %v) {619; CHECK-LABEL: test_vfmsq_lane_f64:620; CHECK: // %bb.0: // %entry621; CHECK-NEXT: // kill: def $d2 killed $d2 def $q2622; CHECK-NEXT: fmls v0.2d, v1.2d, v2.d[0]623; CHECK-NEXT: ret624entry:625 %sub = fsub <1 x double> <double -0.000000e+00>, %v626 %lane = shufflevector <1 x double> %sub, <1 x double> undef, <2 x i32> zeroinitializer627 %0 = tail call <2 x double> @llvm.fma.v2f64(<2 x double> %lane, <2 x double> %b, <2 x double> %a)628 ret <2 x double> %0629}630 631define <2 x double> @test_vfmsq_laneq_f64(<2 x double> %a, <2 x double> %b, <2 x double> %v) {632; CHECK-LABEL: test_vfmsq_laneq_f64:633; CHECK: // %bb.0: // %entry634; CHECK-NEXT: fmls v0.2d, v1.2d, v2.d[1]635; CHECK-NEXT: ret636entry:637 %sub = fsub <2 x double> <double -0.000000e+00, double -0.000000e+00>, %v638 %lane = shufflevector <2 x double> %sub, <2 x double> undef, <2 x i32> <i32 1, i32 1>639 %0 = tail call <2 x double> @llvm.fma.v2f64(<2 x double> %lane, <2 x double> %b, <2 x double> %a)640 ret <2 x double> %0641}642 643define float @test_vfmas_laneq_f32(float %a, float %b, <4 x float> %v) {644; CHECK-LABEL: test_vfmas_laneq_f32:645; CHECK: // %bb.0: // %entry646; CHECK-NEXT: fmla s0, s1, v2.s[3]647; CHECK-NEXT: ret648entry:649 %extract = extractelement <4 x float> %v, i32 3650 %0 = tail call float @llvm.fma.f32(float %b, float %extract, float %a)651 ret float %0652}653 654declare float @llvm.fma.f32(float, float, float)655 656define double @test_vfmsd_lane_f64(double %a, double %b, <1 x double> %v) {657; CHECK-LABEL: test_vfmsd_lane_f64:658; CHECK: // %bb.0: // %entry659; CHECK-NEXT: fmsub d0, d1, d2, d0660; CHECK-NEXT: ret661entry:662 %extract.rhs = extractelement <1 x double> %v, i32 0663 %extract = fsub double -0.000000e+00, %extract.rhs664 %0 = tail call double @llvm.fma.f64(double %b, double %extract, double %a)665 ret double %0666}667 668declare double @llvm.fma.f64(double, double, double)669 670define float @test_vfmss_lane_f32(float %a, float %b, <2 x float> %v) {671; CHECK-LABEL: test_vfmss_lane_f32:672; CHECK: // %bb.0: // %entry673; CHECK-NEXT: // kill: def $d2 killed $d2 def $q2674; CHECK-NEXT: fmls s0, s1, v2.s[1]675; CHECK-NEXT: ret676entry:677 %extract.rhs = extractelement <2 x float> %v, i32 1678 %extract = fsub float -0.000000e+00, %extract.rhs679 %0 = tail call float @llvm.fma.f32(float %b, float %extract, float %a)680 ret float %0681}682 683define float @test_vfmss_laneq_f32(float %a, float %b, <4 x float> %v) {684; CHECK-LABEL: test_vfmss_laneq_f32:685; CHECK: // %bb.0: // %entry686; CHECK-NEXT: fmls s0, s1, v2.s[3]687; CHECK-NEXT: ret688entry:689 %extract.rhs = extractelement <4 x float> %v, i32 3690 %extract = fsub float -0.000000e+00, %extract.rhs691 %0 = tail call float @llvm.fma.f32(float %b, float %extract, float %a)692 ret float %0693}694 695define double @test_vfmsd_laneq_f64(double %a, double %b, <2 x double> %v) {696; CHECK-LABEL: test_vfmsd_laneq_f64:697; CHECK: // %bb.0: // %entry698; CHECK-NEXT: fmls d0, d1, v2.d[1]699; CHECK-NEXT: ret700entry:701 %extract.rhs = extractelement <2 x double> %v, i32 1702 %extract = fsub double -0.000000e+00, %extract.rhs703 %0 = tail call double @llvm.fma.f64(double %b, double %extract, double %a)704 ret double %0705}706 707define double @test_vfmsd_lane_f64_0(double %a, double %b, <1 x double> %v) {708; CHECK-LABEL: test_vfmsd_lane_f64_0:709; CHECK: // %bb.0: // %entry710; CHECK-NEXT: fmsub d0, d1, d2, d0711; CHECK-NEXT: ret712entry:713 %tmp0 = fsub <1 x double> <double -0.000000e+00>, %v714 %tmp1 = extractelement <1 x double> %tmp0, i32 0715 %0 = tail call double @llvm.fma.f64(double %b, double %tmp1, double %a)716 ret double %0717}718 719define float @test_vfmss_lane_f32_0(float %a, float %b, <2 x float> %v) {720; CHECK-LABEL: test_vfmss_lane_f32_0:721; CHECK: // %bb.0: // %entry722; CHECK-NEXT: // kill: def $d2 killed $d2 def $q2723; CHECK-NEXT: fmls s0, s1, v2.s[1]724; CHECK-NEXT: ret725entry:726 %tmp0 = fsub <2 x float> <float -0.000000e+00, float -0.000000e+00>, %v727 %tmp1 = extractelement <2 x float> %tmp0, i32 1728 %0 = tail call float @llvm.fma.f32(float %b, float %tmp1, float %a)729 ret float %0730}731 732define float @test_vfmss_laneq_f32_0(float %a, float %b, <4 x float> %v) {733; CHECK-LABEL: test_vfmss_laneq_f32_0:734; CHECK: // %bb.0: // %entry735; CHECK-NEXT: fmls s0, s1, v2.s[3]736; CHECK-NEXT: ret737entry:738 %tmp0 = fsub <4 x float><float -0.000000e+00, float -0.000000e+00, float -0.000000e+00, float -0.000000e+00>, %v739 %tmp1 = extractelement <4 x float> %tmp0, i32 3740 %0 = tail call float @llvm.fma.f32(float %b, float %tmp1, float %a)741 ret float %0742}743 744define double @test_vfmsd_laneq_f64_0(double %a, double %b, <2 x double> %v) {745; CHECK-LABEL: test_vfmsd_laneq_f64_0:746; CHECK: // %bb.0: // %entry747; CHECK-NEXT: fmls d0, d1, v2.d[1]748; CHECK-NEXT: ret749entry:750 %tmp0 = fsub <2 x double><double -0.000000e+00, double -0.000000e+00>, %v751 %tmp1 = extractelement <2 x double> %tmp0, i32 1752 %0 = tail call double @llvm.fma.f64(double %b, double %tmp1, double %a)753 ret double %0754}755 756define <4 x i32> @test_vmlal_lane_s16(<4 x i32> %a, <4 x i16> %b, <4 x i16> %v) {757; CHECK-LABEL: test_vmlal_lane_s16:758; CHECK: // %bb.0: // %entry759; CHECK-NEXT: // kill: def $d2 killed $d2 def $q2760; CHECK-NEXT: smlal v0.4s, v1.4h, v2.h[3]761; CHECK-NEXT: ret762entry:763 %shuffle = shufflevector <4 x i16> %v, <4 x i16> undef, <4 x i32> <i32 3, i32 3, i32 3, i32 3>764 %vmull2.i = tail call <4 x i32> @llvm.aarch64.neon.smull.v4i32(<4 x i16> %b, <4 x i16> %shuffle)765 %add = add <4 x i32> %vmull2.i, %a766 ret <4 x i32> %add767}768 769define <2 x i64> @test_vmlal_lane_s32(<2 x i64> %a, <2 x i32> %b, <2 x i32> %v) {770; CHECK-LABEL: test_vmlal_lane_s32:771; CHECK: // %bb.0: // %entry772; CHECK-NEXT: // kill: def $d2 killed $d2 def $q2773; CHECK-NEXT: smlal v0.2d, v1.2s, v2.s[1]774; CHECK-NEXT: ret775entry:776 %shuffle = shufflevector <2 x i32> %v, <2 x i32> undef, <2 x i32> <i32 1, i32 1>777 %vmull2.i = tail call <2 x i64> @llvm.aarch64.neon.smull.v2i64(<2 x i32> %b, <2 x i32> %shuffle)778 %add = add <2 x i64> %vmull2.i, %a779 ret <2 x i64> %add780}781 782define <4 x i32> @test_vmlal_laneq_s16(<4 x i32> %a, <4 x i16> %b, <8 x i16> %v) {783; CHECK-LABEL: test_vmlal_laneq_s16:784; CHECK: // %bb.0: // %entry785; CHECK-NEXT: smlal v0.4s, v1.4h, v2.h[7]786; CHECK-NEXT: ret787entry:788 %shuffle = shufflevector <8 x i16> %v, <8 x i16> undef, <4 x i32> <i32 7, i32 7, i32 7, i32 7>789 %vmull2.i = tail call <4 x i32> @llvm.aarch64.neon.smull.v4i32(<4 x i16> %b, <4 x i16> %shuffle)790 %add = add <4 x i32> %vmull2.i, %a791 ret <4 x i32> %add792}793 794define <2 x i64> @test_vmlal_laneq_s32(<2 x i64> %a, <2 x i32> %b, <4 x i32> %v) {795; CHECK-LABEL: test_vmlal_laneq_s32:796; CHECK: // %bb.0: // %entry797; CHECK-NEXT: smlal v0.2d, v1.2s, v2.s[3]798; CHECK-NEXT: ret799entry:800 %shuffle = shufflevector <4 x i32> %v, <4 x i32> undef, <2 x i32> <i32 3, i32 3>801 %vmull2.i = tail call <2 x i64> @llvm.aarch64.neon.smull.v2i64(<2 x i32> %b, <2 x i32> %shuffle)802 %add = add <2 x i64> %vmull2.i, %a803 ret <2 x i64> %add804}805 806define <4 x i32> @test_vmlal_high_lane_s16(<4 x i32> %a, <8 x i16> %b, <4 x i16> %v) {807; CHECK-SD-LABEL: test_vmlal_high_lane_s16:808; CHECK-SD: // %bb.0: // %entry809; CHECK-SD-NEXT: // kill: def $d2 killed $d2 def $q2810; CHECK-SD-NEXT: smlal2 v0.4s, v1.8h, v2.h[3]811; CHECK-SD-NEXT: ret812;813; CHECK-GI-LABEL: test_vmlal_high_lane_s16:814; CHECK-GI: // %bb.0: // %entry815; CHECK-GI-NEXT: mov d1, v1.d[1]816; CHECK-GI-NEXT: // kill: def $d2 killed $d2 def $q2817; CHECK-GI-NEXT: smlal v0.4s, v1.4h, v2.h[3]818; CHECK-GI-NEXT: ret819entry:820 %shuffle.i = shufflevector <8 x i16> %b, <8 x i16> undef, <4 x i32> <i32 4, i32 5, i32 6, i32 7>821 %shuffle = shufflevector <4 x i16> %v, <4 x i16> undef, <4 x i32> <i32 3, i32 3, i32 3, i32 3>822 %vmull2.i = tail call <4 x i32> @llvm.aarch64.neon.smull.v4i32(<4 x i16> %shuffle.i, <4 x i16> %shuffle)823 %add = add <4 x i32> %vmull2.i, %a824 ret <4 x i32> %add825}826 827define <2 x i64> @test_vmlal_high_lane_s32(<2 x i64> %a, <4 x i32> %b, <2 x i32> %v) {828; CHECK-SD-LABEL: test_vmlal_high_lane_s32:829; CHECK-SD: // %bb.0: // %entry830; CHECK-SD-NEXT: // kill: def $d2 killed $d2 def $q2831; CHECK-SD-NEXT: smlal2 v0.2d, v1.4s, v2.s[1]832; CHECK-SD-NEXT: ret833;834; CHECK-GI-LABEL: test_vmlal_high_lane_s32:835; CHECK-GI: // %bb.0: // %entry836; CHECK-GI-NEXT: mov d1, v1.d[1]837; CHECK-GI-NEXT: // kill: def $d2 killed $d2 def $q2838; CHECK-GI-NEXT: smlal v0.2d, v1.2s, v2.s[1]839; CHECK-GI-NEXT: ret840entry:841 %shuffle.i = shufflevector <4 x i32> %b, <4 x i32> undef, <2 x i32> <i32 2, i32 3>842 %shuffle = shufflevector <2 x i32> %v, <2 x i32> undef, <2 x i32> <i32 1, i32 1>843 %vmull2.i = tail call <2 x i64> @llvm.aarch64.neon.smull.v2i64(<2 x i32> %shuffle.i, <2 x i32> %shuffle)844 %add = add <2 x i64> %vmull2.i, %a845 ret <2 x i64> %add846}847 848define <4 x i32> @test_vmlal_high_laneq_s16(<4 x i32> %a, <8 x i16> %b, <8 x i16> %v) {849; CHECK-SD-LABEL: test_vmlal_high_laneq_s16:850; CHECK-SD: // %bb.0: // %entry851; CHECK-SD-NEXT: smlal2 v0.4s, v1.8h, v2.h[7]852; CHECK-SD-NEXT: ret853;854; CHECK-GI-LABEL: test_vmlal_high_laneq_s16:855; CHECK-GI: // %bb.0: // %entry856; CHECK-GI-NEXT: mov d1, v1.d[1]857; CHECK-GI-NEXT: smlal v0.4s, v1.4h, v2.h[7]858; CHECK-GI-NEXT: ret859entry:860 %shuffle.i = shufflevector <8 x i16> %b, <8 x i16> undef, <4 x i32> <i32 4, i32 5, i32 6, i32 7>861 %shuffle = shufflevector <8 x i16> %v, <8 x i16> undef, <4 x i32> <i32 7, i32 7, i32 7, i32 7>862 %vmull2.i = tail call <4 x i32> @llvm.aarch64.neon.smull.v4i32(<4 x i16> %shuffle.i, <4 x i16> %shuffle)863 %add = add <4 x i32> %vmull2.i, %a864 ret <4 x i32> %add865}866 867define <2 x i64> @test_vmlal_high_laneq_s32(<2 x i64> %a, <4 x i32> %b, <4 x i32> %v) {868; CHECK-SD-LABEL: test_vmlal_high_laneq_s32:869; CHECK-SD: // %bb.0: // %entry870; CHECK-SD-NEXT: smlal2 v0.2d, v1.4s, v2.s[3]871; CHECK-SD-NEXT: ret872;873; CHECK-GI-LABEL: test_vmlal_high_laneq_s32:874; CHECK-GI: // %bb.0: // %entry875; CHECK-GI-NEXT: mov d1, v1.d[1]876; CHECK-GI-NEXT: smlal v0.2d, v1.2s, v2.s[3]877; CHECK-GI-NEXT: ret878entry:879 %shuffle.i = shufflevector <4 x i32> %b, <4 x i32> undef, <2 x i32> <i32 2, i32 3>880 %shuffle = shufflevector <4 x i32> %v, <4 x i32> undef, <2 x i32> <i32 3, i32 3>881 %vmull2.i = tail call <2 x i64> @llvm.aarch64.neon.smull.v2i64(<2 x i32> %shuffle.i, <2 x i32> %shuffle)882 %add = add <2 x i64> %vmull2.i, %a883 ret <2 x i64> %add884}885 886define <4 x i32> @test_vmlsl_lane_s16(<4 x i32> %a, <4 x i16> %b, <4 x i16> %v) {887; CHECK-LABEL: test_vmlsl_lane_s16:888; CHECK: // %bb.0: // %entry889; CHECK-NEXT: // kill: def $d2 killed $d2 def $q2890; CHECK-NEXT: smlsl v0.4s, v1.4h, v2.h[3]891; CHECK-NEXT: ret892entry:893 %shuffle = shufflevector <4 x i16> %v, <4 x i16> undef, <4 x i32> <i32 3, i32 3, i32 3, i32 3>894 %vmull2.i = tail call <4 x i32> @llvm.aarch64.neon.smull.v4i32(<4 x i16> %b, <4 x i16> %shuffle)895 %sub = sub <4 x i32> %a, %vmull2.i896 ret <4 x i32> %sub897}898 899define <2 x i64> @test_vmlsl_lane_s32(<2 x i64> %a, <2 x i32> %b, <2 x i32> %v) {900; CHECK-LABEL: test_vmlsl_lane_s32:901; CHECK: // %bb.0: // %entry902; CHECK-NEXT: // kill: def $d2 killed $d2 def $q2903; CHECK-NEXT: smlsl v0.2d, v1.2s, v2.s[1]904; CHECK-NEXT: ret905entry:906 %shuffle = shufflevector <2 x i32> %v, <2 x i32> undef, <2 x i32> <i32 1, i32 1>907 %vmull2.i = tail call <2 x i64> @llvm.aarch64.neon.smull.v2i64(<2 x i32> %b, <2 x i32> %shuffle)908 %sub = sub <2 x i64> %a, %vmull2.i909 ret <2 x i64> %sub910}911 912define <4 x i32> @test_vmlsl_laneq_s16(<4 x i32> %a, <4 x i16> %b, <8 x i16> %v) {913; CHECK-LABEL: test_vmlsl_laneq_s16:914; CHECK: // %bb.0: // %entry915; CHECK-NEXT: smlsl v0.4s, v1.4h, v2.h[7]916; CHECK-NEXT: ret917entry:918 %shuffle = shufflevector <8 x i16> %v, <8 x i16> undef, <4 x i32> <i32 7, i32 7, i32 7, i32 7>919 %vmull2.i = tail call <4 x i32> @llvm.aarch64.neon.smull.v4i32(<4 x i16> %b, <4 x i16> %shuffle)920 %sub = sub <4 x i32> %a, %vmull2.i921 ret <4 x i32> %sub922}923 924define <2 x i64> @test_vmlsl_laneq_s32(<2 x i64> %a, <2 x i32> %b, <4 x i32> %v) {925; CHECK-LABEL: test_vmlsl_laneq_s32:926; CHECK: // %bb.0: // %entry927; CHECK-NEXT: smlsl v0.2d, v1.2s, v2.s[3]928; CHECK-NEXT: ret929entry:930 %shuffle = shufflevector <4 x i32> %v, <4 x i32> undef, <2 x i32> <i32 3, i32 3>931 %vmull2.i = tail call <2 x i64> @llvm.aarch64.neon.smull.v2i64(<2 x i32> %b, <2 x i32> %shuffle)932 %sub = sub <2 x i64> %a, %vmull2.i933 ret <2 x i64> %sub934}935 936define <4 x i32> @test_vmlsl_high_lane_s16(<4 x i32> %a, <8 x i16> %b, <4 x i16> %v) {937; CHECK-SD-LABEL: test_vmlsl_high_lane_s16:938; CHECK-SD: // %bb.0: // %entry939; CHECK-SD-NEXT: // kill: def $d2 killed $d2 def $q2940; CHECK-SD-NEXT: smlsl2 v0.4s, v1.8h, v2.h[3]941; CHECK-SD-NEXT: ret942;943; CHECK-GI-LABEL: test_vmlsl_high_lane_s16:944; CHECK-GI: // %bb.0: // %entry945; CHECK-GI-NEXT: mov d1, v1.d[1]946; CHECK-GI-NEXT: // kill: def $d2 killed $d2 def $q2947; CHECK-GI-NEXT: smlsl v0.4s, v1.4h, v2.h[3]948; CHECK-GI-NEXT: ret949entry:950 %shuffle.i = shufflevector <8 x i16> %b, <8 x i16> undef, <4 x i32> <i32 4, i32 5, i32 6, i32 7>951 %shuffle = shufflevector <4 x i16> %v, <4 x i16> undef, <4 x i32> <i32 3, i32 3, i32 3, i32 3>952 %vmull2.i = tail call <4 x i32> @llvm.aarch64.neon.smull.v4i32(<4 x i16> %shuffle.i, <4 x i16> %shuffle)953 %sub = sub <4 x i32> %a, %vmull2.i954 ret <4 x i32> %sub955}956 957define <2 x i64> @test_vmlsl_high_lane_s32(<2 x i64> %a, <4 x i32> %b, <2 x i32> %v) {958; CHECK-SD-LABEL: test_vmlsl_high_lane_s32:959; CHECK-SD: // %bb.0: // %entry960; CHECK-SD-NEXT: // kill: def $d2 killed $d2 def $q2961; CHECK-SD-NEXT: smlsl2 v0.2d, v1.4s, v2.s[1]962; CHECK-SD-NEXT: ret963;964; CHECK-GI-LABEL: test_vmlsl_high_lane_s32:965; CHECK-GI: // %bb.0: // %entry966; CHECK-GI-NEXT: mov d1, v1.d[1]967; CHECK-GI-NEXT: // kill: def $d2 killed $d2 def $q2968; CHECK-GI-NEXT: smlsl v0.2d, v1.2s, v2.s[1]969; CHECK-GI-NEXT: ret970entry:971 %shuffle.i = shufflevector <4 x i32> %b, <4 x i32> undef, <2 x i32> <i32 2, i32 3>972 %shuffle = shufflevector <2 x i32> %v, <2 x i32> undef, <2 x i32> <i32 1, i32 1>973 %vmull2.i = tail call <2 x i64> @llvm.aarch64.neon.smull.v2i64(<2 x i32> %shuffle.i, <2 x i32> %shuffle)974 %sub = sub <2 x i64> %a, %vmull2.i975 ret <2 x i64> %sub976}977 978define <4 x i32> @test_vmlsl_high_laneq_s16(<4 x i32> %a, <8 x i16> %b, <8 x i16> %v) {979; CHECK-SD-LABEL: test_vmlsl_high_laneq_s16:980; CHECK-SD: // %bb.0: // %entry981; CHECK-SD-NEXT: smlsl2 v0.4s, v1.8h, v2.h[7]982; CHECK-SD-NEXT: ret983;984; CHECK-GI-LABEL: test_vmlsl_high_laneq_s16:985; CHECK-GI: // %bb.0: // %entry986; CHECK-GI-NEXT: mov d1, v1.d[1]987; CHECK-GI-NEXT: smlsl v0.4s, v1.4h, v2.h[7]988; CHECK-GI-NEXT: ret989entry:990 %shuffle.i = shufflevector <8 x i16> %b, <8 x i16> undef, <4 x i32> <i32 4, i32 5, i32 6, i32 7>991 %shuffle = shufflevector <8 x i16> %v, <8 x i16> undef, <4 x i32> <i32 7, i32 7, i32 7, i32 7>992 %vmull2.i = tail call <4 x i32> @llvm.aarch64.neon.smull.v4i32(<4 x i16> %shuffle.i, <4 x i16> %shuffle)993 %sub = sub <4 x i32> %a, %vmull2.i994 ret <4 x i32> %sub995}996 997define <2 x i64> @test_vmlsl_high_laneq_s32(<2 x i64> %a, <4 x i32> %b, <4 x i32> %v) {998; CHECK-SD-LABEL: test_vmlsl_high_laneq_s32:999; CHECK-SD: // %bb.0: // %entry1000; CHECK-SD-NEXT: smlsl2 v0.2d, v1.4s, v2.s[3]1001; CHECK-SD-NEXT: ret1002;1003; CHECK-GI-LABEL: test_vmlsl_high_laneq_s32:1004; CHECK-GI: // %bb.0: // %entry1005; CHECK-GI-NEXT: mov d1, v1.d[1]1006; CHECK-GI-NEXT: smlsl v0.2d, v1.2s, v2.s[3]1007; CHECK-GI-NEXT: ret1008entry:1009 %shuffle.i = shufflevector <4 x i32> %b, <4 x i32> undef, <2 x i32> <i32 2, i32 3>1010 %shuffle = shufflevector <4 x i32> %v, <4 x i32> undef, <2 x i32> <i32 3, i32 3>1011 %vmull2.i = tail call <2 x i64> @llvm.aarch64.neon.smull.v2i64(<2 x i32> %shuffle.i, <2 x i32> %shuffle)1012 %sub = sub <2 x i64> %a, %vmull2.i1013 ret <2 x i64> %sub1014}1015 1016define <4 x i32> @test_vmlal_lane_u16(<4 x i32> %a, <4 x i16> %b, <4 x i16> %v) {1017; CHECK-LABEL: test_vmlal_lane_u16:1018; CHECK: // %bb.0: // %entry1019; CHECK-NEXT: // kill: def $d2 killed $d2 def $q21020; CHECK-NEXT: umlal v0.4s, v1.4h, v2.h[3]1021; CHECK-NEXT: ret1022entry:1023 %shuffle = shufflevector <4 x i16> %v, <4 x i16> undef, <4 x i32> <i32 3, i32 3, i32 3, i32 3>1024 %vmull2.i = tail call <4 x i32> @llvm.aarch64.neon.umull.v4i32(<4 x i16> %b, <4 x i16> %shuffle)1025 %add = add <4 x i32> %vmull2.i, %a1026 ret <4 x i32> %add1027}1028 1029define <2 x i64> @test_vmlal_lane_u32(<2 x i64> %a, <2 x i32> %b, <2 x i32> %v) {1030; CHECK-LABEL: test_vmlal_lane_u32:1031; CHECK: // %bb.0: // %entry1032; CHECK-NEXT: // kill: def $d2 killed $d2 def $q21033; CHECK-NEXT: umlal v0.2d, v1.2s, v2.s[1]1034; CHECK-NEXT: ret1035entry:1036 %shuffle = shufflevector <2 x i32> %v, <2 x i32> undef, <2 x i32> <i32 1, i32 1>1037 %vmull2.i = tail call <2 x i64> @llvm.aarch64.neon.umull.v2i64(<2 x i32> %b, <2 x i32> %shuffle)1038 %add = add <2 x i64> %vmull2.i, %a1039 ret <2 x i64> %add1040}1041 1042define <4 x i32> @test_vmlal_laneq_u16(<4 x i32> %a, <4 x i16> %b, <8 x i16> %v) {1043; CHECK-LABEL: test_vmlal_laneq_u16:1044; CHECK: // %bb.0: // %entry1045; CHECK-NEXT: umlal v0.4s, v1.4h, v2.h[7]1046; CHECK-NEXT: ret1047entry:1048 %shuffle = shufflevector <8 x i16> %v, <8 x i16> undef, <4 x i32> <i32 7, i32 7, i32 7, i32 7>1049 %vmull2.i = tail call <4 x i32> @llvm.aarch64.neon.umull.v4i32(<4 x i16> %b, <4 x i16> %shuffle)1050 %add = add <4 x i32> %vmull2.i, %a1051 ret <4 x i32> %add1052}1053 1054define <2 x i64> @test_vmlal_laneq_u32(<2 x i64> %a, <2 x i32> %b, <4 x i32> %v) {1055; CHECK-LABEL: test_vmlal_laneq_u32:1056; CHECK: // %bb.0: // %entry1057; CHECK-NEXT: umlal v0.2d, v1.2s, v2.s[3]1058; CHECK-NEXT: ret1059entry:1060 %shuffle = shufflevector <4 x i32> %v, <4 x i32> undef, <2 x i32> <i32 3, i32 3>1061 %vmull2.i = tail call <2 x i64> @llvm.aarch64.neon.umull.v2i64(<2 x i32> %b, <2 x i32> %shuffle)1062 %add = add <2 x i64> %vmull2.i, %a1063 ret <2 x i64> %add1064}1065 1066define <4 x i32> @test_vmlal_high_lane_u16(<4 x i32> %a, <8 x i16> %b, <4 x i16> %v) {1067; CHECK-SD-LABEL: test_vmlal_high_lane_u16:1068; CHECK-SD: // %bb.0: // %entry1069; CHECK-SD-NEXT: // kill: def $d2 killed $d2 def $q21070; CHECK-SD-NEXT: umlal2 v0.4s, v1.8h, v2.h[3]1071; CHECK-SD-NEXT: ret1072;1073; CHECK-GI-LABEL: test_vmlal_high_lane_u16:1074; CHECK-GI: // %bb.0: // %entry1075; CHECK-GI-NEXT: mov d1, v1.d[1]1076; CHECK-GI-NEXT: // kill: def $d2 killed $d2 def $q21077; CHECK-GI-NEXT: umlal v0.4s, v1.4h, v2.h[3]1078; CHECK-GI-NEXT: ret1079entry:1080 %shuffle.i = shufflevector <8 x i16> %b, <8 x i16> undef, <4 x i32> <i32 4, i32 5, i32 6, i32 7>1081 %shuffle = shufflevector <4 x i16> %v, <4 x i16> undef, <4 x i32> <i32 3, i32 3, i32 3, i32 3>1082 %vmull2.i = tail call <4 x i32> @llvm.aarch64.neon.umull.v4i32(<4 x i16> %shuffle.i, <4 x i16> %shuffle)1083 %add = add <4 x i32> %vmull2.i, %a1084 ret <4 x i32> %add1085}1086 1087define <2 x i64> @test_vmlal_high_lane_u32(<2 x i64> %a, <4 x i32> %b, <2 x i32> %v) {1088; CHECK-SD-LABEL: test_vmlal_high_lane_u32:1089; CHECK-SD: // %bb.0: // %entry1090; CHECK-SD-NEXT: // kill: def $d2 killed $d2 def $q21091; CHECK-SD-NEXT: umlal2 v0.2d, v1.4s, v2.s[1]1092; CHECK-SD-NEXT: ret1093;1094; CHECK-GI-LABEL: test_vmlal_high_lane_u32:1095; CHECK-GI: // %bb.0: // %entry1096; CHECK-GI-NEXT: mov d1, v1.d[1]1097; CHECK-GI-NEXT: // kill: def $d2 killed $d2 def $q21098; CHECK-GI-NEXT: umlal v0.2d, v1.2s, v2.s[1]1099; CHECK-GI-NEXT: ret1100entry:1101 %shuffle.i = shufflevector <4 x i32> %b, <4 x i32> undef, <2 x i32> <i32 2, i32 3>1102 %shuffle = shufflevector <2 x i32> %v, <2 x i32> undef, <2 x i32> <i32 1, i32 1>1103 %vmull2.i = tail call <2 x i64> @llvm.aarch64.neon.umull.v2i64(<2 x i32> %shuffle.i, <2 x i32> %shuffle)1104 %add = add <2 x i64> %vmull2.i, %a1105 ret <2 x i64> %add1106}1107 1108define <4 x i32> @test_vmlal_high_laneq_u16(<4 x i32> %a, <8 x i16> %b, <8 x i16> %v) {1109; CHECK-SD-LABEL: test_vmlal_high_laneq_u16:1110; CHECK-SD: // %bb.0: // %entry1111; CHECK-SD-NEXT: umlal2 v0.4s, v1.8h, v2.h[7]1112; CHECK-SD-NEXT: ret1113;1114; CHECK-GI-LABEL: test_vmlal_high_laneq_u16:1115; CHECK-GI: // %bb.0: // %entry1116; CHECK-GI-NEXT: mov d1, v1.d[1]1117; CHECK-GI-NEXT: umlal v0.4s, v1.4h, v2.h[7]1118; CHECK-GI-NEXT: ret1119entry:1120 %shuffle.i = shufflevector <8 x i16> %b, <8 x i16> undef, <4 x i32> <i32 4, i32 5, i32 6, i32 7>1121 %shuffle = shufflevector <8 x i16> %v, <8 x i16> undef, <4 x i32> <i32 7, i32 7, i32 7, i32 7>1122 %vmull2.i = tail call <4 x i32> @llvm.aarch64.neon.umull.v4i32(<4 x i16> %shuffle.i, <4 x i16> %shuffle)1123 %add = add <4 x i32> %vmull2.i, %a1124 ret <4 x i32> %add1125}1126 1127define <2 x i64> @test_vmlal_high_laneq_u32(<2 x i64> %a, <4 x i32> %b, <4 x i32> %v) {1128; CHECK-SD-LABEL: test_vmlal_high_laneq_u32:1129; CHECK-SD: // %bb.0: // %entry1130; CHECK-SD-NEXT: umlal2 v0.2d, v1.4s, v2.s[3]1131; CHECK-SD-NEXT: ret1132;1133; CHECK-GI-LABEL: test_vmlal_high_laneq_u32:1134; CHECK-GI: // %bb.0: // %entry1135; CHECK-GI-NEXT: mov d1, v1.d[1]1136; CHECK-GI-NEXT: umlal v0.2d, v1.2s, v2.s[3]1137; CHECK-GI-NEXT: ret1138entry:1139 %shuffle.i = shufflevector <4 x i32> %b, <4 x i32> undef, <2 x i32> <i32 2, i32 3>1140 %shuffle = shufflevector <4 x i32> %v, <4 x i32> undef, <2 x i32> <i32 3, i32 3>1141 %vmull2.i = tail call <2 x i64> @llvm.aarch64.neon.umull.v2i64(<2 x i32> %shuffle.i, <2 x i32> %shuffle)1142 %add = add <2 x i64> %vmull2.i, %a1143 ret <2 x i64> %add1144}1145 1146define <4 x i32> @test_vmlsl_lane_u16(<4 x i32> %a, <4 x i16> %b, <4 x i16> %v) {1147; CHECK-LABEL: test_vmlsl_lane_u16:1148; CHECK: // %bb.0: // %entry1149; CHECK-NEXT: // kill: def $d2 killed $d2 def $q21150; CHECK-NEXT: umlsl v0.4s, v1.4h, v2.h[3]1151; CHECK-NEXT: ret1152entry:1153 %shuffle = shufflevector <4 x i16> %v, <4 x i16> undef, <4 x i32> <i32 3, i32 3, i32 3, i32 3>1154 %vmull2.i = tail call <4 x i32> @llvm.aarch64.neon.umull.v4i32(<4 x i16> %b, <4 x i16> %shuffle)1155 %sub = sub <4 x i32> %a, %vmull2.i1156 ret <4 x i32> %sub1157}1158 1159define <2 x i64> @test_vmlsl_lane_u32(<2 x i64> %a, <2 x i32> %b, <2 x i32> %v) {1160; CHECK-LABEL: test_vmlsl_lane_u32:1161; CHECK: // %bb.0: // %entry1162; CHECK-NEXT: // kill: def $d2 killed $d2 def $q21163; CHECK-NEXT: umlsl v0.2d, v1.2s, v2.s[1]1164; CHECK-NEXT: ret1165entry:1166 %shuffle = shufflevector <2 x i32> %v, <2 x i32> undef, <2 x i32> <i32 1, i32 1>1167 %vmull2.i = tail call <2 x i64> @llvm.aarch64.neon.umull.v2i64(<2 x i32> %b, <2 x i32> %shuffle)1168 %sub = sub <2 x i64> %a, %vmull2.i1169 ret <2 x i64> %sub1170}1171 1172define <4 x i32> @test_vmlsl_laneq_u16(<4 x i32> %a, <4 x i16> %b, <8 x i16> %v) {1173; CHECK-LABEL: test_vmlsl_laneq_u16:1174; CHECK: // %bb.0: // %entry1175; CHECK-NEXT: umlsl v0.4s, v1.4h, v2.h[7]1176; CHECK-NEXT: ret1177entry:1178 %shuffle = shufflevector <8 x i16> %v, <8 x i16> undef, <4 x i32> <i32 7, i32 7, i32 7, i32 7>1179 %vmull2.i = tail call <4 x i32> @llvm.aarch64.neon.umull.v4i32(<4 x i16> %b, <4 x i16> %shuffle)1180 %sub = sub <4 x i32> %a, %vmull2.i1181 ret <4 x i32> %sub1182}1183 1184define <2 x i64> @test_vmlsl_laneq_u32(<2 x i64> %a, <2 x i32> %b, <4 x i32> %v) {1185; CHECK-LABEL: test_vmlsl_laneq_u32:1186; CHECK: // %bb.0: // %entry1187; CHECK-NEXT: umlsl v0.2d, v1.2s, v2.s[3]1188; CHECK-NEXT: ret1189entry:1190 %shuffle = shufflevector <4 x i32> %v, <4 x i32> undef, <2 x i32> <i32 3, i32 3>1191 %vmull2.i = tail call <2 x i64> @llvm.aarch64.neon.umull.v2i64(<2 x i32> %b, <2 x i32> %shuffle)1192 %sub = sub <2 x i64> %a, %vmull2.i1193 ret <2 x i64> %sub1194}1195 1196define <4 x i32> @test_vmlsl_high_lane_u16(<4 x i32> %a, <8 x i16> %b, <4 x i16> %v) {1197; CHECK-SD-LABEL: test_vmlsl_high_lane_u16:1198; CHECK-SD: // %bb.0: // %entry1199; CHECK-SD-NEXT: // kill: def $d2 killed $d2 def $q21200; CHECK-SD-NEXT: umlsl2 v0.4s, v1.8h, v2.h[3]1201; CHECK-SD-NEXT: ret1202;1203; CHECK-GI-LABEL: test_vmlsl_high_lane_u16:1204; CHECK-GI: // %bb.0: // %entry1205; CHECK-GI-NEXT: mov d1, v1.d[1]1206; CHECK-GI-NEXT: // kill: def $d2 killed $d2 def $q21207; CHECK-GI-NEXT: umlsl v0.4s, v1.4h, v2.h[3]1208; CHECK-GI-NEXT: ret1209entry:1210 %shuffle.i = shufflevector <8 x i16> %b, <8 x i16> undef, <4 x i32> <i32 4, i32 5, i32 6, i32 7>1211 %shuffle = shufflevector <4 x i16> %v, <4 x i16> undef, <4 x i32> <i32 3, i32 3, i32 3, i32 3>1212 %vmull2.i = tail call <4 x i32> @llvm.aarch64.neon.umull.v4i32(<4 x i16> %shuffle.i, <4 x i16> %shuffle)1213 %sub = sub <4 x i32> %a, %vmull2.i1214 ret <4 x i32> %sub1215}1216 1217define <2 x i64> @test_vmlsl_high_lane_u32(<2 x i64> %a, <4 x i32> %b, <2 x i32> %v) {1218; CHECK-SD-LABEL: test_vmlsl_high_lane_u32:1219; CHECK-SD: // %bb.0: // %entry1220; CHECK-SD-NEXT: // kill: def $d2 killed $d2 def $q21221; CHECK-SD-NEXT: umlsl2 v0.2d, v1.4s, v2.s[1]1222; CHECK-SD-NEXT: ret1223;1224; CHECK-GI-LABEL: test_vmlsl_high_lane_u32:1225; CHECK-GI: // %bb.0: // %entry1226; CHECK-GI-NEXT: mov d1, v1.d[1]1227; CHECK-GI-NEXT: // kill: def $d2 killed $d2 def $q21228; CHECK-GI-NEXT: umlsl v0.2d, v1.2s, v2.s[1]1229; CHECK-GI-NEXT: ret1230entry:1231 %shuffle.i = shufflevector <4 x i32> %b, <4 x i32> undef, <2 x i32> <i32 2, i32 3>1232 %shuffle = shufflevector <2 x i32> %v, <2 x i32> undef, <2 x i32> <i32 1, i32 1>1233 %vmull2.i = tail call <2 x i64> @llvm.aarch64.neon.umull.v2i64(<2 x i32> %shuffle.i, <2 x i32> %shuffle)1234 %sub = sub <2 x i64> %a, %vmull2.i1235 ret <2 x i64> %sub1236}1237 1238define <4 x i32> @test_vmlsl_high_laneq_u16(<4 x i32> %a, <8 x i16> %b, <8 x i16> %v) {1239; CHECK-SD-LABEL: test_vmlsl_high_laneq_u16:1240; CHECK-SD: // %bb.0: // %entry1241; CHECK-SD-NEXT: umlsl2 v0.4s, v1.8h, v2.h[7]1242; CHECK-SD-NEXT: ret1243;1244; CHECK-GI-LABEL: test_vmlsl_high_laneq_u16:1245; CHECK-GI: // %bb.0: // %entry1246; CHECK-GI-NEXT: mov d1, v1.d[1]1247; CHECK-GI-NEXT: umlsl v0.4s, v1.4h, v2.h[7]1248; CHECK-GI-NEXT: ret1249entry:1250 %shuffle.i = shufflevector <8 x i16> %b, <8 x i16> undef, <4 x i32> <i32 4, i32 5, i32 6, i32 7>1251 %shuffle = shufflevector <8 x i16> %v, <8 x i16> undef, <4 x i32> <i32 7, i32 7, i32 7, i32 7>1252 %vmull2.i = tail call <4 x i32> @llvm.aarch64.neon.umull.v4i32(<4 x i16> %shuffle.i, <4 x i16> %shuffle)1253 %sub = sub <4 x i32> %a, %vmull2.i1254 ret <4 x i32> %sub1255}1256 1257define <2 x i64> @test_vmlsl_high_laneq_u32(<2 x i64> %a, <4 x i32> %b, <4 x i32> %v) {1258; CHECK-SD-LABEL: test_vmlsl_high_laneq_u32:1259; CHECK-SD: // %bb.0: // %entry1260; CHECK-SD-NEXT: umlsl2 v0.2d, v1.4s, v2.s[3]1261; CHECK-SD-NEXT: ret1262;1263; CHECK-GI-LABEL: test_vmlsl_high_laneq_u32:1264; CHECK-GI: // %bb.0: // %entry1265; CHECK-GI-NEXT: mov d1, v1.d[1]1266; CHECK-GI-NEXT: umlsl v0.2d, v1.2s, v2.s[3]1267; CHECK-GI-NEXT: ret1268entry:1269 %shuffle.i = shufflevector <4 x i32> %b, <4 x i32> undef, <2 x i32> <i32 2, i32 3>1270 %shuffle = shufflevector <4 x i32> %v, <4 x i32> undef, <2 x i32> <i32 3, i32 3>1271 %vmull2.i = tail call <2 x i64> @llvm.aarch64.neon.umull.v2i64(<2 x i32> %shuffle.i, <2 x i32> %shuffle)1272 %sub = sub <2 x i64> %a, %vmull2.i1273 ret <2 x i64> %sub1274}1275 1276define <4 x i32> @test_vmull_lane_s16(<4 x i16> %a, <4 x i16> %v) {1277; CHECK-LABEL: test_vmull_lane_s16:1278; CHECK: // %bb.0: // %entry1279; CHECK-NEXT: // kill: def $d1 killed $d1 def $q11280; CHECK-NEXT: smull v0.4s, v0.4h, v1.h[3]1281; CHECK-NEXT: ret1282entry:1283 %shuffle = shufflevector <4 x i16> %v, <4 x i16> undef, <4 x i32> <i32 3, i32 3, i32 3, i32 3>1284 %vmull2.i = tail call <4 x i32> @llvm.aarch64.neon.smull.v4i32(<4 x i16> %a, <4 x i16> %shuffle)1285 ret <4 x i32> %vmull2.i1286}1287 1288define <2 x i64> @test_vmull_lane_s32(<2 x i32> %a, <2 x i32> %v) {1289; CHECK-LABEL: test_vmull_lane_s32:1290; CHECK: // %bb.0: // %entry1291; CHECK-NEXT: // kill: def $d1 killed $d1 def $q11292; CHECK-NEXT: smull v0.2d, v0.2s, v1.s[1]1293; CHECK-NEXT: ret1294entry:1295 %shuffle = shufflevector <2 x i32> %v, <2 x i32> undef, <2 x i32> <i32 1, i32 1>1296 %vmull2.i = tail call <2 x i64> @llvm.aarch64.neon.smull.v2i64(<2 x i32> %a, <2 x i32> %shuffle)1297 ret <2 x i64> %vmull2.i1298}1299 1300define <4 x i32> @test_vmull_lane_u16(<4 x i16> %a, <4 x i16> %v) {1301; CHECK-LABEL: test_vmull_lane_u16:1302; CHECK: // %bb.0: // %entry1303; CHECK-NEXT: // kill: def $d1 killed $d1 def $q11304; CHECK-NEXT: umull v0.4s, v0.4h, v1.h[3]1305; CHECK-NEXT: ret1306entry:1307 %shuffle = shufflevector <4 x i16> %v, <4 x i16> undef, <4 x i32> <i32 3, i32 3, i32 3, i32 3>1308 %vmull2.i = tail call <4 x i32> @llvm.aarch64.neon.umull.v4i32(<4 x i16> %a, <4 x i16> %shuffle)1309 ret <4 x i32> %vmull2.i1310}1311 1312define <2 x i64> @test_vmull_lane_u32(<2 x i32> %a, <2 x i32> %v) {1313; CHECK-LABEL: test_vmull_lane_u32:1314; CHECK: // %bb.0: // %entry1315; CHECK-NEXT: // kill: def $d1 killed $d1 def $q11316; CHECK-NEXT: umull v0.2d, v0.2s, v1.s[1]1317; CHECK-NEXT: ret1318entry:1319 %shuffle = shufflevector <2 x i32> %v, <2 x i32> undef, <2 x i32> <i32 1, i32 1>1320 %vmull2.i = tail call <2 x i64> @llvm.aarch64.neon.umull.v2i64(<2 x i32> %a, <2 x i32> %shuffle)1321 ret <2 x i64> %vmull2.i1322}1323 1324define <4 x i32> @test_vmull_high_lane_s16(<8 x i16> %a, <4 x i16> %v) {1325; CHECK-SD-LABEL: test_vmull_high_lane_s16:1326; CHECK-SD: // %bb.0: // %entry1327; CHECK-SD-NEXT: // kill: def $d1 killed $d1 def $q11328; CHECK-SD-NEXT: smull2 v0.4s, v0.8h, v1.h[3]1329; CHECK-SD-NEXT: ret1330;1331; CHECK-GI-LABEL: test_vmull_high_lane_s16:1332; CHECK-GI: // %bb.0: // %entry1333; CHECK-GI-NEXT: mov d0, v0.d[1]1334; CHECK-GI-NEXT: // kill: def $d1 killed $d1 def $q11335; CHECK-GI-NEXT: smull v0.4s, v0.4h, v1.h[3]1336; CHECK-GI-NEXT: ret1337entry:1338 %shuffle.i = shufflevector <8 x i16> %a, <8 x i16> undef, <4 x i32> <i32 4, i32 5, i32 6, i32 7>1339 %shuffle = shufflevector <4 x i16> %v, <4 x i16> undef, <4 x i32> <i32 3, i32 3, i32 3, i32 3>1340 %vmull2.i = tail call <4 x i32> @llvm.aarch64.neon.smull.v4i32(<4 x i16> %shuffle.i, <4 x i16> %shuffle)1341 ret <4 x i32> %vmull2.i1342}1343 1344define <2 x i64> @test_vmull_high_lane_s32(<4 x i32> %a, <2 x i32> %v) {1345; CHECK-SD-LABEL: test_vmull_high_lane_s32:1346; CHECK-SD: // %bb.0: // %entry1347; CHECK-SD-NEXT: // kill: def $d1 killed $d1 def $q11348; CHECK-SD-NEXT: smull2 v0.2d, v0.4s, v1.s[1]1349; CHECK-SD-NEXT: ret1350;1351; CHECK-GI-LABEL: test_vmull_high_lane_s32:1352; CHECK-GI: // %bb.0: // %entry1353; CHECK-GI-NEXT: mov d0, v0.d[1]1354; CHECK-GI-NEXT: // kill: def $d1 killed $d1 def $q11355; CHECK-GI-NEXT: smull v0.2d, v0.2s, v1.s[1]1356; CHECK-GI-NEXT: ret1357entry:1358 %shuffle.i = shufflevector <4 x i32> %a, <4 x i32> undef, <2 x i32> <i32 2, i32 3>1359 %shuffle = shufflevector <2 x i32> %v, <2 x i32> undef, <2 x i32> <i32 1, i32 1>1360 %vmull2.i = tail call <2 x i64> @llvm.aarch64.neon.smull.v2i64(<2 x i32> %shuffle.i, <2 x i32> %shuffle)1361 ret <2 x i64> %vmull2.i1362}1363 1364define <4 x i32> @test_vmull_high_lane_u16(<8 x i16> %a, <4 x i16> %v) {1365; CHECK-SD-LABEL: test_vmull_high_lane_u16:1366; CHECK-SD: // %bb.0: // %entry1367; CHECK-SD-NEXT: // kill: def $d1 killed $d1 def $q11368; CHECK-SD-NEXT: umull2 v0.4s, v0.8h, v1.h[3]1369; CHECK-SD-NEXT: ret1370;1371; CHECK-GI-LABEL: test_vmull_high_lane_u16:1372; CHECK-GI: // %bb.0: // %entry1373; CHECK-GI-NEXT: mov d0, v0.d[1]1374; CHECK-GI-NEXT: // kill: def $d1 killed $d1 def $q11375; CHECK-GI-NEXT: umull v0.4s, v0.4h, v1.h[3]1376; CHECK-GI-NEXT: ret1377entry:1378 %shuffle.i = shufflevector <8 x i16> %a, <8 x i16> undef, <4 x i32> <i32 4, i32 5, i32 6, i32 7>1379 %shuffle = shufflevector <4 x i16> %v, <4 x i16> undef, <4 x i32> <i32 3, i32 3, i32 3, i32 3>1380 %vmull2.i = tail call <4 x i32> @llvm.aarch64.neon.umull.v4i32(<4 x i16> %shuffle.i, <4 x i16> %shuffle)1381 ret <4 x i32> %vmull2.i1382}1383 1384define <2 x i64> @test_vmull_high_lane_u32(<4 x i32> %a, <2 x i32> %v) {1385; CHECK-SD-LABEL: test_vmull_high_lane_u32:1386; CHECK-SD: // %bb.0: // %entry1387; CHECK-SD-NEXT: // kill: def $d1 killed $d1 def $q11388; CHECK-SD-NEXT: umull2 v0.2d, v0.4s, v1.s[1]1389; CHECK-SD-NEXT: ret1390;1391; CHECK-GI-LABEL: test_vmull_high_lane_u32:1392; CHECK-GI: // %bb.0: // %entry1393; CHECK-GI-NEXT: mov d0, v0.d[1]1394; CHECK-GI-NEXT: // kill: def $d1 killed $d1 def $q11395; CHECK-GI-NEXT: umull v0.2d, v0.2s, v1.s[1]1396; CHECK-GI-NEXT: ret1397entry:1398 %shuffle.i = shufflevector <4 x i32> %a, <4 x i32> undef, <2 x i32> <i32 2, i32 3>1399 %shuffle = shufflevector <2 x i32> %v, <2 x i32> undef, <2 x i32> <i32 1, i32 1>1400 %vmull2.i = tail call <2 x i64> @llvm.aarch64.neon.umull.v2i64(<2 x i32> %shuffle.i, <2 x i32> %shuffle)1401 ret <2 x i64> %vmull2.i1402}1403 1404define <4 x i32> @test_vmull_laneq_s16(<4 x i16> %a, <8 x i16> %v) {1405; CHECK-LABEL: test_vmull_laneq_s16:1406; CHECK: // %bb.0: // %entry1407; CHECK-NEXT: smull v0.4s, v0.4h, v1.h[7]1408; CHECK-NEXT: ret1409entry:1410 %shuffle = shufflevector <8 x i16> %v, <8 x i16> undef, <4 x i32> <i32 7, i32 7, i32 7, i32 7>1411 %vmull2.i = tail call <4 x i32> @llvm.aarch64.neon.smull.v4i32(<4 x i16> %a, <4 x i16> %shuffle)1412 ret <4 x i32> %vmull2.i1413}1414 1415define <2 x i64> @test_vmull_laneq_s32(<2 x i32> %a, <4 x i32> %v) {1416; CHECK-LABEL: test_vmull_laneq_s32:1417; CHECK: // %bb.0: // %entry1418; CHECK-NEXT: smull v0.2d, v0.2s, v1.s[3]1419; CHECK-NEXT: ret1420entry:1421 %shuffle = shufflevector <4 x i32> %v, <4 x i32> undef, <2 x i32> <i32 3, i32 3>1422 %vmull2.i = tail call <2 x i64> @llvm.aarch64.neon.smull.v2i64(<2 x i32> %a, <2 x i32> %shuffle)1423 ret <2 x i64> %vmull2.i1424}1425 1426define <4 x i32> @test_vmull_laneq_u16(<4 x i16> %a, <8 x i16> %v) {1427; CHECK-LABEL: test_vmull_laneq_u16:1428; CHECK: // %bb.0: // %entry1429; CHECK-NEXT: umull v0.4s, v0.4h, v1.h[7]1430; CHECK-NEXT: ret1431entry:1432 %shuffle = shufflevector <8 x i16> %v, <8 x i16> undef, <4 x i32> <i32 7, i32 7, i32 7, i32 7>1433 %vmull2.i = tail call <4 x i32> @llvm.aarch64.neon.umull.v4i32(<4 x i16> %a, <4 x i16> %shuffle)1434 ret <4 x i32> %vmull2.i1435}1436 1437define <2 x i64> @test_vmull_laneq_u32(<2 x i32> %a, <4 x i32> %v) {1438; CHECK-LABEL: test_vmull_laneq_u32:1439; CHECK: // %bb.0: // %entry1440; CHECK-NEXT: umull v0.2d, v0.2s, v1.s[3]1441; CHECK-NEXT: ret1442entry:1443 %shuffle = shufflevector <4 x i32> %v, <4 x i32> undef, <2 x i32> <i32 3, i32 3>1444 %vmull2.i = tail call <2 x i64> @llvm.aarch64.neon.umull.v2i64(<2 x i32> %a, <2 x i32> %shuffle)1445 ret <2 x i64> %vmull2.i1446}1447 1448define <4 x i32> @test_vmull_high_laneq_s16(<8 x i16> %a, <8 x i16> %v) {1449; CHECK-SD-LABEL: test_vmull_high_laneq_s16:1450; CHECK-SD: // %bb.0: // %entry1451; CHECK-SD-NEXT: smull2 v0.4s, v0.8h, v1.h[7]1452; CHECK-SD-NEXT: ret1453;1454; CHECK-GI-LABEL: test_vmull_high_laneq_s16:1455; CHECK-GI: // %bb.0: // %entry1456; CHECK-GI-NEXT: mov d0, v0.d[1]1457; CHECK-GI-NEXT: smull v0.4s, v0.4h, v1.h[7]1458; CHECK-GI-NEXT: ret1459entry:1460 %shuffle.i = shufflevector <8 x i16> %a, <8 x i16> undef, <4 x i32> <i32 4, i32 5, i32 6, i32 7>1461 %shuffle = shufflevector <8 x i16> %v, <8 x i16> undef, <4 x i32> <i32 7, i32 7, i32 7, i32 7>1462 %vmull2.i = tail call <4 x i32> @llvm.aarch64.neon.smull.v4i32(<4 x i16> %shuffle.i, <4 x i16> %shuffle)1463 ret <4 x i32> %vmull2.i1464}1465 1466define <2 x i64> @test_vmull_high_laneq_s32(<4 x i32> %a, <4 x i32> %v) {1467; CHECK-SD-LABEL: test_vmull_high_laneq_s32:1468; CHECK-SD: // %bb.0: // %entry1469; CHECK-SD-NEXT: smull2 v0.2d, v0.4s, v1.s[3]1470; CHECK-SD-NEXT: ret1471;1472; CHECK-GI-LABEL: test_vmull_high_laneq_s32:1473; CHECK-GI: // %bb.0: // %entry1474; CHECK-GI-NEXT: mov d0, v0.d[1]1475; CHECK-GI-NEXT: smull v0.2d, v0.2s, v1.s[3]1476; CHECK-GI-NEXT: ret1477entry:1478 %shuffle.i = shufflevector <4 x i32> %a, <4 x i32> undef, <2 x i32> <i32 2, i32 3>1479 %shuffle = shufflevector <4 x i32> %v, <4 x i32> undef, <2 x i32> <i32 3, i32 3>1480 %vmull2.i = tail call <2 x i64> @llvm.aarch64.neon.smull.v2i64(<2 x i32> %shuffle.i, <2 x i32> %shuffle)1481 ret <2 x i64> %vmull2.i1482}1483 1484define <4 x i32> @test_vmull_high_laneq_u16(<8 x i16> %a, <8 x i16> %v) {1485; CHECK-SD-LABEL: test_vmull_high_laneq_u16:1486; CHECK-SD: // %bb.0: // %entry1487; CHECK-SD-NEXT: umull2 v0.4s, v0.8h, v1.h[7]1488; CHECK-SD-NEXT: ret1489;1490; CHECK-GI-LABEL: test_vmull_high_laneq_u16:1491; CHECK-GI: // %bb.0: // %entry1492; CHECK-GI-NEXT: mov d0, v0.d[1]1493; CHECK-GI-NEXT: umull v0.4s, v0.4h, v1.h[7]1494; CHECK-GI-NEXT: ret1495entry:1496 %shuffle.i = shufflevector <8 x i16> %a, <8 x i16> undef, <4 x i32> <i32 4, i32 5, i32 6, i32 7>1497 %shuffle = shufflevector <8 x i16> %v, <8 x i16> undef, <4 x i32> <i32 7, i32 7, i32 7, i32 7>1498 %vmull2.i = tail call <4 x i32> @llvm.aarch64.neon.umull.v4i32(<4 x i16> %shuffle.i, <4 x i16> %shuffle)1499 ret <4 x i32> %vmull2.i1500}1501 1502define <2 x i64> @test_vmull_high_laneq_u32(<4 x i32> %a, <4 x i32> %v) {1503; CHECK-SD-LABEL: test_vmull_high_laneq_u32:1504; CHECK-SD: // %bb.0: // %entry1505; CHECK-SD-NEXT: umull2 v0.2d, v0.4s, v1.s[3]1506; CHECK-SD-NEXT: ret1507;1508; CHECK-GI-LABEL: test_vmull_high_laneq_u32:1509; CHECK-GI: // %bb.0: // %entry1510; CHECK-GI-NEXT: mov d0, v0.d[1]1511; CHECK-GI-NEXT: umull v0.2d, v0.2s, v1.s[3]1512; CHECK-GI-NEXT: ret1513entry:1514 %shuffle.i = shufflevector <4 x i32> %a, <4 x i32> undef, <2 x i32> <i32 2, i32 3>1515 %shuffle = shufflevector <4 x i32> %v, <4 x i32> undef, <2 x i32> <i32 3, i32 3>1516 %vmull2.i = tail call <2 x i64> @llvm.aarch64.neon.umull.v2i64(<2 x i32> %shuffle.i, <2 x i32> %shuffle)1517 ret <2 x i64> %vmull2.i1518}1519 1520define <4 x i32> @test_vqdmlal_lane_s16(<4 x i32> %a, <4 x i16> %b, <4 x i16> %v) {1521; CHECK-LABEL: test_vqdmlal_lane_s16:1522; CHECK: // %bb.0: // %entry1523; CHECK-NEXT: // kill: def $d2 killed $d2 def $q21524; CHECK-NEXT: sqdmlal v0.4s, v1.4h, v2.h[3]1525; CHECK-NEXT: ret1526entry:1527 %shuffle = shufflevector <4 x i16> %v, <4 x i16> undef, <4 x i32> <i32 3, i32 3, i32 3, i32 3>1528 %vqdmlal2.i = tail call <4 x i32> @llvm.aarch64.neon.sqdmull.v4i32(<4 x i16> %b, <4 x i16> %shuffle)1529 %vqdmlal4.i = tail call <4 x i32> @llvm.aarch64.neon.sqadd.v4i32(<4 x i32> %a, <4 x i32> %vqdmlal2.i)1530 ret <4 x i32> %vqdmlal4.i1531}1532 1533define <2 x i64> @test_vqdmlal_lane_s32(<2 x i64> %a, <2 x i32> %b, <2 x i32> %v) {1534; CHECK-LABEL: test_vqdmlal_lane_s32:1535; CHECK: // %bb.0: // %entry1536; CHECK-NEXT: // kill: def $d2 killed $d2 def $q21537; CHECK-NEXT: sqdmlal v0.2d, v1.2s, v2.s[1]1538; CHECK-NEXT: ret1539entry:1540 %shuffle = shufflevector <2 x i32> %v, <2 x i32> undef, <2 x i32> <i32 1, i32 1>1541 %vqdmlal2.i = tail call <2 x i64> @llvm.aarch64.neon.sqdmull.v2i64(<2 x i32> %b, <2 x i32> %shuffle)1542 %vqdmlal4.i = tail call <2 x i64> @llvm.aarch64.neon.sqadd.v2i64(<2 x i64> %a, <2 x i64> %vqdmlal2.i)1543 ret <2 x i64> %vqdmlal4.i1544}1545 1546define <4 x i32> @test_vqdmlal_high_lane_s16(<4 x i32> %a, <8 x i16> %b, <4 x i16> %v) {1547; CHECK-SD-LABEL: test_vqdmlal_high_lane_s16:1548; CHECK-SD: // %bb.0: // %entry1549; CHECK-SD-NEXT: // kill: def $d2 killed $d2 def $q21550; CHECK-SD-NEXT: sqdmlal2 v0.4s, v1.8h, v2.h[3]1551; CHECK-SD-NEXT: ret1552;1553; CHECK-GI-LABEL: test_vqdmlal_high_lane_s16:1554; CHECK-GI: // %bb.0: // %entry1555; CHECK-GI-NEXT: mov d1, v1.d[1]1556; CHECK-GI-NEXT: // kill: def $d2 killed $d2 def $q21557; CHECK-GI-NEXT: sqdmlal v0.4s, v1.4h, v2.h[3]1558; CHECK-GI-NEXT: ret1559entry:1560 %shuffle.i = shufflevector <8 x i16> %b, <8 x i16> undef, <4 x i32> <i32 4, i32 5, i32 6, i32 7>1561 %shuffle = shufflevector <4 x i16> %v, <4 x i16> undef, <4 x i32> <i32 3, i32 3, i32 3, i32 3>1562 %vqdmlal2.i = tail call <4 x i32> @llvm.aarch64.neon.sqdmull.v4i32(<4 x i16> %shuffle.i, <4 x i16> %shuffle)1563 %vqdmlal4.i = tail call <4 x i32> @llvm.aarch64.neon.sqadd.v4i32(<4 x i32> %a, <4 x i32> %vqdmlal2.i)1564 ret <4 x i32> %vqdmlal4.i1565}1566 1567define <2 x i64> @test_vqdmlal_high_lane_s32(<2 x i64> %a, <4 x i32> %b, <2 x i32> %v) {1568; CHECK-SD-LABEL: test_vqdmlal_high_lane_s32:1569; CHECK-SD: // %bb.0: // %entry1570; CHECK-SD-NEXT: // kill: def $d2 killed $d2 def $q21571; CHECK-SD-NEXT: sqdmlal2 v0.2d, v1.4s, v2.s[1]1572; CHECK-SD-NEXT: ret1573;1574; CHECK-GI-LABEL: test_vqdmlal_high_lane_s32:1575; CHECK-GI: // %bb.0: // %entry1576; CHECK-GI-NEXT: mov d1, v1.d[1]1577; CHECK-GI-NEXT: // kill: def $d2 killed $d2 def $q21578; CHECK-GI-NEXT: sqdmlal v0.2d, v1.2s, v2.s[1]1579; CHECK-GI-NEXT: ret1580entry:1581 %shuffle.i = shufflevector <4 x i32> %b, <4 x i32> undef, <2 x i32> <i32 2, i32 3>1582 %shuffle = shufflevector <2 x i32> %v, <2 x i32> undef, <2 x i32> <i32 1, i32 1>1583 %vqdmlal2.i = tail call <2 x i64> @llvm.aarch64.neon.sqdmull.v2i64(<2 x i32> %shuffle.i, <2 x i32> %shuffle)1584 %vqdmlal4.i = tail call <2 x i64> @llvm.aarch64.neon.sqadd.v2i64(<2 x i64> %a, <2 x i64> %vqdmlal2.i)1585 ret <2 x i64> %vqdmlal4.i1586}1587 1588define <4 x i32> @test_vqdmlsl_lane_s16(<4 x i32> %a, <4 x i16> %b, <4 x i16> %v) {1589; CHECK-LABEL: test_vqdmlsl_lane_s16:1590; CHECK: // %bb.0: // %entry1591; CHECK-NEXT: // kill: def $d2 killed $d2 def $q21592; CHECK-NEXT: sqdmlsl v0.4s, v1.4h, v2.h[3]1593; CHECK-NEXT: ret1594entry:1595 %shuffle = shufflevector <4 x i16> %v, <4 x i16> undef, <4 x i32> <i32 3, i32 3, i32 3, i32 3>1596 %vqdmlsl2.i = tail call <4 x i32> @llvm.aarch64.neon.sqdmull.v4i32(<4 x i16> %b, <4 x i16> %shuffle)1597 %vqdmlsl4.i = tail call <4 x i32> @llvm.aarch64.neon.sqsub.v4i32(<4 x i32> %a, <4 x i32> %vqdmlsl2.i)1598 ret <4 x i32> %vqdmlsl4.i1599}1600 1601define <2 x i64> @test_vqdmlsl_lane_s32(<2 x i64> %a, <2 x i32> %b, <2 x i32> %v) {1602; CHECK-LABEL: test_vqdmlsl_lane_s32:1603; CHECK: // %bb.0: // %entry1604; CHECK-NEXT: // kill: def $d2 killed $d2 def $q21605; CHECK-NEXT: sqdmlsl v0.2d, v1.2s, v2.s[1]1606; CHECK-NEXT: ret1607entry:1608 %shuffle = shufflevector <2 x i32> %v, <2 x i32> undef, <2 x i32> <i32 1, i32 1>1609 %vqdmlsl2.i = tail call <2 x i64> @llvm.aarch64.neon.sqdmull.v2i64(<2 x i32> %b, <2 x i32> %shuffle)1610 %vqdmlsl4.i = tail call <2 x i64> @llvm.aarch64.neon.sqsub.v2i64(<2 x i64> %a, <2 x i64> %vqdmlsl2.i)1611 ret <2 x i64> %vqdmlsl4.i1612}1613 1614define <4 x i32> @test_vqdmlsl_high_lane_s16(<4 x i32> %a, <8 x i16> %b, <4 x i16> %v) {1615; CHECK-SD-LABEL: test_vqdmlsl_high_lane_s16:1616; CHECK-SD: // %bb.0: // %entry1617; CHECK-SD-NEXT: // kill: def $d2 killed $d2 def $q21618; CHECK-SD-NEXT: sqdmlsl2 v0.4s, v1.8h, v2.h[3]1619; CHECK-SD-NEXT: ret1620;1621; CHECK-GI-LABEL: test_vqdmlsl_high_lane_s16:1622; CHECK-GI: // %bb.0: // %entry1623; CHECK-GI-NEXT: mov d1, v1.d[1]1624; CHECK-GI-NEXT: // kill: def $d2 killed $d2 def $q21625; CHECK-GI-NEXT: sqdmlsl v0.4s, v1.4h, v2.h[3]1626; CHECK-GI-NEXT: ret1627entry:1628 %shuffle.i = shufflevector <8 x i16> %b, <8 x i16> undef, <4 x i32> <i32 4, i32 5, i32 6, i32 7>1629 %shuffle = shufflevector <4 x i16> %v, <4 x i16> undef, <4 x i32> <i32 3, i32 3, i32 3, i32 3>1630 %vqdmlsl2.i = tail call <4 x i32> @llvm.aarch64.neon.sqdmull.v4i32(<4 x i16> %shuffle.i, <4 x i16> %shuffle)1631 %vqdmlsl4.i = tail call <4 x i32> @llvm.aarch64.neon.sqsub.v4i32(<4 x i32> %a, <4 x i32> %vqdmlsl2.i)1632 ret <4 x i32> %vqdmlsl4.i1633}1634 1635define <2 x i64> @test_vqdmlsl_high_lane_s32(<2 x i64> %a, <4 x i32> %b, <2 x i32> %v) {1636; CHECK-SD-LABEL: test_vqdmlsl_high_lane_s32:1637; CHECK-SD: // %bb.0: // %entry1638; CHECK-SD-NEXT: // kill: def $d2 killed $d2 def $q21639; CHECK-SD-NEXT: sqdmlsl2 v0.2d, v1.4s, v2.s[1]1640; CHECK-SD-NEXT: ret1641;1642; CHECK-GI-LABEL: test_vqdmlsl_high_lane_s32:1643; CHECK-GI: // %bb.0: // %entry1644; CHECK-GI-NEXT: mov d1, v1.d[1]1645; CHECK-GI-NEXT: // kill: def $d2 killed $d2 def $q21646; CHECK-GI-NEXT: sqdmlsl v0.2d, v1.2s, v2.s[1]1647; CHECK-GI-NEXT: ret1648entry:1649 %shuffle.i = shufflevector <4 x i32> %b, <4 x i32> undef, <2 x i32> <i32 2, i32 3>1650 %shuffle = shufflevector <2 x i32> %v, <2 x i32> undef, <2 x i32> <i32 1, i32 1>1651 %vqdmlsl2.i = tail call <2 x i64> @llvm.aarch64.neon.sqdmull.v2i64(<2 x i32> %shuffle.i, <2 x i32> %shuffle)1652 %vqdmlsl4.i = tail call <2 x i64> @llvm.aarch64.neon.sqsub.v2i64(<2 x i64> %a, <2 x i64> %vqdmlsl2.i)1653 ret <2 x i64> %vqdmlsl4.i1654}1655 1656define <4 x i32> @test_vqdmull_lane_s16(<4 x i16> %a, <4 x i16> %v) {1657; CHECK-LABEL: test_vqdmull_lane_s16:1658; CHECK: // %bb.0: // %entry1659; CHECK-NEXT: // kill: def $d1 killed $d1 def $q11660; CHECK-NEXT: sqdmull v0.4s, v0.4h, v1.h[3]1661; CHECK-NEXT: ret1662entry:1663 %shuffle = shufflevector <4 x i16> %v, <4 x i16> undef, <4 x i32> <i32 3, i32 3, i32 3, i32 3>1664 %vqdmull2.i = tail call <4 x i32> @llvm.aarch64.neon.sqdmull.v4i32(<4 x i16> %a, <4 x i16> %shuffle)1665 ret <4 x i32> %vqdmull2.i1666}1667 1668define <2 x i64> @test_vqdmull_lane_s32(<2 x i32> %a, <2 x i32> %v) {1669; CHECK-LABEL: test_vqdmull_lane_s32:1670; CHECK: // %bb.0: // %entry1671; CHECK-NEXT: // kill: def $d1 killed $d1 def $q11672; CHECK-NEXT: sqdmull v0.2d, v0.2s, v1.s[1]1673; CHECK-NEXT: ret1674entry:1675 %shuffle = shufflevector <2 x i32> %v, <2 x i32> undef, <2 x i32> <i32 1, i32 1>1676 %vqdmull2.i = tail call <2 x i64> @llvm.aarch64.neon.sqdmull.v2i64(<2 x i32> %a, <2 x i32> %shuffle)1677 ret <2 x i64> %vqdmull2.i1678}1679 1680define <4 x i32> @test_vqdmull_laneq_s16(<4 x i16> %a, <8 x i16> %v) {1681; CHECK-LABEL: test_vqdmull_laneq_s16:1682; CHECK: // %bb.0: // %entry1683; CHECK-NEXT: sqdmull v0.4s, v0.4h, v1.h[3]1684; CHECK-NEXT: ret1685entry:1686 %shuffle = shufflevector <8 x i16> %v, <8 x i16> undef, <4 x i32> <i32 3, i32 3, i32 3, i32 3>1687 %vqdmull2.i = tail call <4 x i32> @llvm.aarch64.neon.sqdmull.v4i32(<4 x i16> %a, <4 x i16> %shuffle)1688 ret <4 x i32> %vqdmull2.i1689}1690 1691define <2 x i64> @test_vqdmull_laneq_s32(<2 x i32> %a, <4 x i32> %v) {1692; CHECK-LABEL: test_vqdmull_laneq_s32:1693; CHECK: // %bb.0: // %entry1694; CHECK-NEXT: sqdmull v0.2d, v0.2s, v1.s[3]1695; CHECK-NEXT: ret1696entry:1697 %shuffle = shufflevector <4 x i32> %v, <4 x i32> undef, <2 x i32> <i32 3, i32 3>1698 %vqdmull2.i = tail call <2 x i64> @llvm.aarch64.neon.sqdmull.v2i64(<2 x i32> %a, <2 x i32> %shuffle)1699 ret <2 x i64> %vqdmull2.i1700}1701 1702define <4 x i32> @test_vqdmull_high_lane_s16(<8 x i16> %a, <4 x i16> %v) {1703; CHECK-SD-LABEL: test_vqdmull_high_lane_s16:1704; CHECK-SD: // %bb.0: // %entry1705; CHECK-SD-NEXT: // kill: def $d1 killed $d1 def $q11706; CHECK-SD-NEXT: sqdmull2 v0.4s, v0.8h, v1.h[3]1707; CHECK-SD-NEXT: ret1708;1709; CHECK-GI-LABEL: test_vqdmull_high_lane_s16:1710; CHECK-GI: // %bb.0: // %entry1711; CHECK-GI-NEXT: mov d0, v0.d[1]1712; CHECK-GI-NEXT: // kill: def $d1 killed $d1 def $q11713; CHECK-GI-NEXT: sqdmull v0.4s, v0.4h, v1.h[3]1714; CHECK-GI-NEXT: ret1715entry:1716 %shuffle.i = shufflevector <8 x i16> %a, <8 x i16> undef, <4 x i32> <i32 4, i32 5, i32 6, i32 7>1717 %shuffle = shufflevector <4 x i16> %v, <4 x i16> undef, <4 x i32> <i32 3, i32 3, i32 3, i32 3>1718 %vqdmull2.i = tail call <4 x i32> @llvm.aarch64.neon.sqdmull.v4i32(<4 x i16> %shuffle.i, <4 x i16> %shuffle)1719 ret <4 x i32> %vqdmull2.i1720}1721 1722define <2 x i64> @test_vqdmull_high_lane_s32(<4 x i32> %a, <2 x i32> %v) {1723; CHECK-SD-LABEL: test_vqdmull_high_lane_s32:1724; CHECK-SD: // %bb.0: // %entry1725; CHECK-SD-NEXT: // kill: def $d1 killed $d1 def $q11726; CHECK-SD-NEXT: sqdmull2 v0.2d, v0.4s, v1.s[1]1727; CHECK-SD-NEXT: ret1728;1729; CHECK-GI-LABEL: test_vqdmull_high_lane_s32:1730; CHECK-GI: // %bb.0: // %entry1731; CHECK-GI-NEXT: mov d0, v0.d[1]1732; CHECK-GI-NEXT: // kill: def $d1 killed $d1 def $q11733; CHECK-GI-NEXT: sqdmull v0.2d, v0.2s, v1.s[1]1734; CHECK-GI-NEXT: ret1735entry:1736 %shuffle.i = shufflevector <4 x i32> %a, <4 x i32> undef, <2 x i32> <i32 2, i32 3>1737 %shuffle = shufflevector <2 x i32> %v, <2 x i32> undef, <2 x i32> <i32 1, i32 1>1738 %vqdmull2.i = tail call <2 x i64> @llvm.aarch64.neon.sqdmull.v2i64(<2 x i32> %shuffle.i, <2 x i32> %shuffle)1739 ret <2 x i64> %vqdmull2.i1740}1741 1742define <4 x i32> @test_vqdmull_high_laneq_s16(<8 x i16> %a, <8 x i16> %v) {1743; CHECK-SD-LABEL: test_vqdmull_high_laneq_s16:1744; CHECK-SD: // %bb.0: // %entry1745; CHECK-SD-NEXT: sqdmull2 v0.4s, v0.8h, v1.h[7]1746; CHECK-SD-NEXT: ret1747;1748; CHECK-GI-LABEL: test_vqdmull_high_laneq_s16:1749; CHECK-GI: // %bb.0: // %entry1750; CHECK-GI-NEXT: mov d0, v0.d[1]1751; CHECK-GI-NEXT: sqdmull v0.4s, v0.4h, v1.h[7]1752; CHECK-GI-NEXT: ret1753entry:1754 %shuffle.i = shufflevector <8 x i16> %a, <8 x i16> undef, <4 x i32> <i32 4, i32 5, i32 6, i32 7>1755 %shuffle = shufflevector <8 x i16> %v, <8 x i16> undef, <4 x i32> <i32 7, i32 7, i32 7, i32 7>1756 %vqdmull2.i = tail call <4 x i32> @llvm.aarch64.neon.sqdmull.v4i32(<4 x i16> %shuffle.i, <4 x i16> %shuffle)1757 ret <4 x i32> %vqdmull2.i1758}1759 1760define <2 x i64> @test_vqdmull_high_laneq_s32(<4 x i32> %a, <4 x i32> %v) {1761; CHECK-SD-LABEL: test_vqdmull_high_laneq_s32:1762; CHECK-SD: // %bb.0: // %entry1763; CHECK-SD-NEXT: sqdmull2 v0.2d, v0.4s, v1.s[3]1764; CHECK-SD-NEXT: ret1765;1766; CHECK-GI-LABEL: test_vqdmull_high_laneq_s32:1767; CHECK-GI: // %bb.0: // %entry1768; CHECK-GI-NEXT: mov d0, v0.d[1]1769; CHECK-GI-NEXT: sqdmull v0.2d, v0.2s, v1.s[3]1770; CHECK-GI-NEXT: ret1771entry:1772 %shuffle.i = shufflevector <4 x i32> %a, <4 x i32> undef, <2 x i32> <i32 2, i32 3>1773 %shuffle = shufflevector <4 x i32> %v, <4 x i32> undef, <2 x i32> <i32 3, i32 3>1774 %vqdmull2.i = tail call <2 x i64> @llvm.aarch64.neon.sqdmull.v2i64(<2 x i32> %shuffle.i, <2 x i32> %shuffle)1775 ret <2 x i64> %vqdmull2.i1776}1777 1778define <4 x i16> @test_vqdmulh_lane_s16(<4 x i16> %a, <4 x i16> %v) {1779; CHECK-LABEL: test_vqdmulh_lane_s16:1780; CHECK: // %bb.0: // %entry1781; CHECK-NEXT: // kill: def $d1 killed $d1 def $q11782; CHECK-NEXT: sqdmulh v0.4h, v0.4h, v1.h[3]1783; CHECK-NEXT: ret1784entry:1785 %shuffle = shufflevector <4 x i16> %v, <4 x i16> undef, <4 x i32> <i32 3, i32 3, i32 3, i32 3>1786 %vqdmulh2.i = tail call <4 x i16> @llvm.aarch64.neon.sqdmulh.v4i16(<4 x i16> %a, <4 x i16> %shuffle)1787 ret <4 x i16> %vqdmulh2.i1788}1789 1790define <4 x i16> @test_vqdmulh_lane_s16_intrinsic(<4 x i16> %a, <4 x i16> %v) {1791; CHECK-LABEL: test_vqdmulh_lane_s16_intrinsic:1792; CHECK: // %bb.0: // %entry1793; CHECK-NEXT: // kill: def $d1 killed $d1 def $q11794; CHECK-NEXT: sqdmulh v0.4h, v0.4h, v1.h[3]1795; CHECK-NEXT: ret1796entry:1797 %vqdmulh2.i = tail call <4 x i16> @llvm.aarch64.neon.sqdmulh.lane.v4i16.v4i16(<4 x i16> %a, <4 x i16> %v, i32 3)1798 ret <4 x i16> %vqdmulh2.i1799}1800 1801define <4 x i16> @test_vqdmulh_laneq_s16_intrinsic_lo(<4 x i16> %a, <8 x i16> %v) {1802; CHECK-LABEL: test_vqdmulh_laneq_s16_intrinsic_lo:1803; CHECK: // %bb.0: // %entry1804; CHECK-NEXT: sqdmulh v0.4h, v0.4h, v1.h[3]1805; CHECK-NEXT: ret1806entry:1807 %vqdmulh2.i = tail call <4 x i16> @llvm.aarch64.neon.sqdmulh.laneq.v4i16.v8i16(<4 x i16> %a, <8 x i16> %v, i32 3)1808 ret <4 x i16> %vqdmulh2.i1809}1810 1811define <4 x i16> @test_vqdmulh_laneq_s16_intrinsic_hi(<4 x i16> %a, <8 x i16> %v) {1812; CHECK-LABEL: test_vqdmulh_laneq_s16_intrinsic_hi:1813; CHECK: // %bb.0: // %entry1814; CHECK-NEXT: sqdmulh v0.4h, v0.4h, v1.h[7]1815; CHECK-NEXT: ret1816entry:1817 %vqdmulh2.i = tail call <4 x i16> @llvm.aarch64.neon.sqdmulh.laneq.v4i16.v8i16(<4 x i16> %a, <8 x i16> %v, i32 7)1818 ret <4 x i16> %vqdmulh2.i1819}1820 1821define <8 x i16> @test_vqdmulhq_lane_s16(<8 x i16> %a, <4 x i16> %v) {1822; CHECK-LABEL: test_vqdmulhq_lane_s16:1823; CHECK: // %bb.0: // %entry1824; CHECK-NEXT: // kill: def $d1 killed $d1 def $q11825; CHECK-NEXT: sqdmulh v0.8h, v0.8h, v1.h[3]1826; CHECK-NEXT: ret1827entry:1828 %shuffle = shufflevector <4 x i16> %v, <4 x i16> undef, <8 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>1829 %vqdmulh2.i = tail call <8 x i16> @llvm.aarch64.neon.sqdmulh.v8i16(<8 x i16> %a, <8 x i16> %shuffle)1830 ret <8 x i16> %vqdmulh2.i1831}1832 1833define <8 x i16> @test_vqdmulhq_lane_s16_intrinsic(<8 x i16> %a, <4 x i16> %v) {1834; CHECK-LABEL: test_vqdmulhq_lane_s16_intrinsic:1835; CHECK: // %bb.0: // %entry1836; CHECK-NEXT: // kill: def $d1 killed $d1 def $q11837; CHECK-NEXT: sqdmulh v0.8h, v0.8h, v1.h[3]1838; CHECK-NEXT: ret1839entry:1840 %vqdmulh2.i = tail call <8 x i16> @llvm.aarch64.neon.sqdmulh.lane.v8i16.v4i16(<8 x i16> %a, <4 x i16> %v, i32 3)1841 ret <8 x i16> %vqdmulh2.i1842}1843 1844define <8 x i16> @test_vqdmulhq_laneq_s16_intrinsic_lo(<8 x i16> %a, <8 x i16> %v) {1845; CHECK-LABEL: test_vqdmulhq_laneq_s16_intrinsic_lo:1846; CHECK: // %bb.0: // %entry1847; CHECK-NEXT: sqdmulh v0.8h, v0.8h, v1.h[3]1848; CHECK-NEXT: ret1849entry:1850 %vqdmulh2.i = tail call <8 x i16> @llvm.aarch64.neon.sqdmulh.laneq.v8i16.v8i16(<8 x i16> %a, <8 x i16> %v, i32 3)1851 ret <8 x i16> %vqdmulh2.i1852}1853 1854define <8 x i16> @test_vqdmulhq_laneq_s16_intrinsic_hi(<8 x i16> %a, <8 x i16> %v) {1855; CHECK-LABEL: test_vqdmulhq_laneq_s16_intrinsic_hi:1856; CHECK: // %bb.0: // %entry1857; CHECK-NEXT: sqdmulh v0.8h, v0.8h, v1.h[7]1858; CHECK-NEXT: ret1859entry:1860 %vqdmulh2.i = tail call <8 x i16> @llvm.aarch64.neon.sqdmulh.laneq.v8i16.v8i16(<8 x i16> %a, <8 x i16> %v, i32 7)1861 ret <8 x i16> %vqdmulh2.i1862}1863 1864define <2 x i32> @test_vqdmulh_lane_s32(<2 x i32> %a, <2 x i32> %v) {1865; CHECK-LABEL: test_vqdmulh_lane_s32:1866; CHECK: // %bb.0: // %entry1867; CHECK-NEXT: // kill: def $d1 killed $d1 def $q11868; CHECK-NEXT: sqdmulh v0.2s, v0.2s, v1.s[1]1869; CHECK-NEXT: ret1870entry:1871 %shuffle = shufflevector <2 x i32> %v, <2 x i32> undef, <2 x i32> <i32 1, i32 1>1872 %vqdmulh2.i = tail call <2 x i32> @llvm.aarch64.neon.sqdmulh.v2i32(<2 x i32> %a, <2 x i32> %shuffle)1873 ret <2 x i32> %vqdmulh2.i1874}1875 1876define <2 x i32> @test_vqdmulh_lane_s32_intrinsic(<2 x i32> %a, <2 x i32> %v) {1877; CHECK-LABEL: test_vqdmulh_lane_s32_intrinsic:1878; CHECK: // %bb.0: // %entry1879; CHECK-NEXT: // kill: def $d1 killed $d1 def $q11880; CHECK-NEXT: sqdmulh v0.2s, v0.2s, v1.s[1]1881; CHECK-NEXT: ret1882entry:1883 %vqdmulh2.i = tail call <2 x i32> @llvm.aarch64.neon.sqdmulh.lane.v2i32.v2i32(<2 x i32> %a, <2 x i32> %v, i32 1)1884 ret <2 x i32> %vqdmulh2.i1885}1886 1887define <2 x i32> @test_vqdmulh_laneq_s32_intrinsic_lo(<2 x i32> %a, <4 x i32> %v) {1888; CHECK-LABEL: test_vqdmulh_laneq_s32_intrinsic_lo:1889; CHECK: // %bb.0: // %entry1890; CHECK-NEXT: sqdmulh v0.2s, v0.2s, v1.s[1]1891; CHECK-NEXT: ret1892entry:1893 %vqdmulh2.i = tail call <2 x i32> @llvm.aarch64.neon.sqdmulh.laneq.v2i32.v4i32(<2 x i32> %a, <4 x i32> %v, i32 1)1894 ret <2 x i32> %vqdmulh2.i1895}1896 1897define <2 x i32> @test_vqdmulh_laneq_s32_intrinsic_hi(<2 x i32> %a, <4 x i32> %v) {1898; CHECK-LABEL: test_vqdmulh_laneq_s32_intrinsic_hi:1899; CHECK: // %bb.0: // %entry1900; CHECK-NEXT: sqdmulh v0.2s, v0.2s, v1.s[3]1901; CHECK-NEXT: ret1902entry:1903 %vqdmulh2.i = tail call <2 x i32> @llvm.aarch64.neon.sqdmulh.laneq.v2i32.v4i32(<2 x i32> %a, <4 x i32> %v, i32 3)1904 ret <2 x i32> %vqdmulh2.i1905}1906 1907define <4 x i32> @test_vqdmulhq_lane_s32(<4 x i32> %a, <2 x i32> %v) {1908; CHECK-LABEL: test_vqdmulhq_lane_s32:1909; CHECK: // %bb.0: // %entry1910; CHECK-NEXT: // kill: def $d1 killed $d1 def $q11911; CHECK-NEXT: sqdmulh v0.4s, v0.4s, v1.s[1]1912; CHECK-NEXT: ret1913entry:1914 %shuffle = shufflevector <2 x i32> %v, <2 x i32> undef, <4 x i32> <i32 1, i32 1, i32 1, i32 1>1915 %vqdmulh2.i = tail call <4 x i32> @llvm.aarch64.neon.sqdmulh.v4i32(<4 x i32> %a, <4 x i32> %shuffle)1916 ret <4 x i32> %vqdmulh2.i1917}1918 1919define <4 x i32> @test_vqdmulhq_lane_s32_intrinsic(<4 x i32> %a, <2 x i32> %v) {1920; CHECK-LABEL: test_vqdmulhq_lane_s32_intrinsic:1921; CHECK: // %bb.0: // %entry1922; CHECK-NEXT: // kill: def $d1 killed $d1 def $q11923; CHECK-NEXT: sqdmulh v0.4s, v0.4s, v1.s[1]1924; CHECK-NEXT: ret1925entry:1926 %vqdmulh2.i = tail call <4 x i32> @llvm.aarch64.neon.sqdmulh.lane.v4i32.v2i32(<4 x i32> %a, <2 x i32> %v, i32 1)1927 ret <4 x i32> %vqdmulh2.i1928}1929 1930define <4 x i32> @test_vqdmulhq_laneq_s32_intrinsic_lo(<4 x i32> %a, <4 x i32> %v) {1931; CHECK-LABEL: test_vqdmulhq_laneq_s32_intrinsic_lo:1932; CHECK: // %bb.0: // %entry1933; CHECK-NEXT: sqdmulh v0.4s, v0.4s, v1.s[1]1934; CHECK-NEXT: ret1935entry:1936 %vqdmulh2.i = tail call <4 x i32> @llvm.aarch64.neon.sqdmulh.laneq.v4i32.v4i32(<4 x i32> %a, <4 x i32> %v, i32 1)1937 ret <4 x i32> %vqdmulh2.i1938}1939 1940define <4 x i32> @test_vqdmulhq_laneq_s32_intrinsic_hi(<4 x i32> %a, <4 x i32> %v) {1941; CHECK-LABEL: test_vqdmulhq_laneq_s32_intrinsic_hi:1942; CHECK: // %bb.0: // %entry1943; CHECK-NEXT: sqdmulh v0.4s, v0.4s, v1.s[3]1944; CHECK-NEXT: ret1945entry:1946 %vqdmulh2.i = tail call <4 x i32> @llvm.aarch64.neon.sqdmulh.laneq.v4i32.v4i32(<4 x i32> %a, <4 x i32> %v, i32 3)1947 ret <4 x i32> %vqdmulh2.i1948}1949 1950define <4 x i16> @test_vqrdmulh_lane_s16(<4 x i16> %a, <4 x i16> %v) {1951; CHECK-LABEL: test_vqrdmulh_lane_s16:1952; CHECK: // %bb.0: // %entry1953; CHECK-NEXT: // kill: def $d1 killed $d1 def $q11954; CHECK-NEXT: sqrdmulh v0.4h, v0.4h, v1.h[3]1955; CHECK-NEXT: ret1956entry:1957 %shuffle = shufflevector <4 x i16> %v, <4 x i16> undef, <4 x i32> <i32 3, i32 3, i32 3, i32 3>1958 %vqrdmulh2.i = tail call <4 x i16> @llvm.aarch64.neon.sqrdmulh.v4i16(<4 x i16> %a, <4 x i16> %shuffle)1959 ret <4 x i16> %vqrdmulh2.i1960}1961 1962define <4 x i16> @test_vqrdmulh_lane_s16_intrinsic(<4 x i16> %a, <4 x i16> %v) {1963; CHECK-LABEL: test_vqrdmulh_lane_s16_intrinsic:1964; CHECK: // %bb.0: // %entry1965; CHECK-NEXT: // kill: def $d1 killed $d1 def $q11966; CHECK-NEXT: sqrdmulh v0.4h, v0.4h, v1.h[3]1967; CHECK-NEXT: ret1968entry:1969 %vqrdmulh2.i = tail call <4 x i16> @llvm.aarch64.neon.sqrdmulh.lane.v4i16.v4i16(<4 x i16> %a, <4 x i16> %v, i32 3)1970 ret <4 x i16> %vqrdmulh2.i1971}1972 1973define <4 x i16> @test_vqrdmulh_laneq_s16_intrinsic_lo(<4 x i16> %a, <8 x i16> %v) {1974; CHECK-LABEL: test_vqrdmulh_laneq_s16_intrinsic_lo:1975; CHECK: // %bb.0: // %entry1976; CHECK-NEXT: sqrdmulh v0.4h, v0.4h, v1.h[3]1977; CHECK-NEXT: ret1978entry:1979 %vqrdmulh2.i = tail call <4 x i16> @llvm.aarch64.neon.sqrdmulh.laneq.v4i16.v8i16(<4 x i16> %a, <8 x i16> %v, i32 3)1980 ret <4 x i16> %vqrdmulh2.i1981}1982 1983define <4 x i16> @test_vqrdmulh_laneq_s16_intrinsic_hi(<4 x i16> %a, <8 x i16> %v) {1984; CHECK-LABEL: test_vqrdmulh_laneq_s16_intrinsic_hi:1985; CHECK: // %bb.0: // %entry1986; CHECK-NEXT: sqrdmulh v0.4h, v0.4h, v1.h[7]1987; CHECK-NEXT: ret1988entry:1989 %vqrdmulh2.i = tail call <4 x i16> @llvm.aarch64.neon.sqrdmulh.laneq.v4i16.v8i16(<4 x i16> %a, <8 x i16> %v, i32 7)1990 ret <4 x i16> %vqrdmulh2.i1991}1992 1993define <8 x i16> @test_vqrdmulhq_lane_s16(<8 x i16> %a, <4 x i16> %v) {1994; CHECK-LABEL: test_vqrdmulhq_lane_s16:1995; CHECK: // %bb.0: // %entry1996; CHECK-NEXT: // kill: def $d1 killed $d1 def $q11997; CHECK-NEXT: sqrdmulh v0.8h, v0.8h, v1.h[3]1998; CHECK-NEXT: ret1999entry:2000 %shuffle = shufflevector <4 x i16> %v, <4 x i16> undef, <8 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>2001 %vqrdmulh2.i = tail call <8 x i16> @llvm.aarch64.neon.sqrdmulh.v8i16(<8 x i16> %a, <8 x i16> %shuffle)2002 ret <8 x i16> %vqrdmulh2.i2003}2004 2005define <8 x i16> @test_vqrdmulhq_lane_s16_intrinsic(<8 x i16> %a, <4 x i16> %v) {2006; CHECK-LABEL: test_vqrdmulhq_lane_s16_intrinsic:2007; CHECK: // %bb.0: // %entry2008; CHECK-NEXT: // kill: def $d1 killed $d1 def $q12009; CHECK-NEXT: sqrdmulh v0.8h, v0.8h, v1.h[3]2010; CHECK-NEXT: ret2011entry:2012 %vqrdmulh2.i = tail call <8 x i16> @llvm.aarch64.neon.sqrdmulh.lane.v8i16.v4i16(<8 x i16> %a, <4 x i16> %v, i32 3)2013 ret <8 x i16> %vqrdmulh2.i2014}2015 2016define <8 x i16> @test_vqrdmulhq_laneq_s16_intrinsic_lo(<8 x i16> %a, <8 x i16> %v) {2017; CHECK-LABEL: test_vqrdmulhq_laneq_s16_intrinsic_lo:2018; CHECK: // %bb.0: // %entry2019; CHECK-NEXT: sqrdmulh v0.8h, v0.8h, v1.h[3]2020; CHECK-NEXT: ret2021entry:2022 %vqrdmulh2.i = tail call <8 x i16> @llvm.aarch64.neon.sqrdmulh.laneq.v8i16.v8i16(<8 x i16> %a, <8 x i16> %v, i32 3)2023 ret <8 x i16> %vqrdmulh2.i2024}2025 2026define <8 x i16> @test_vqrdmulhq_laneq_s16_intrinsic_hi(<8 x i16> %a, <8 x i16> %v) {2027; CHECK-LABEL: test_vqrdmulhq_laneq_s16_intrinsic_hi:2028; CHECK: // %bb.0: // %entry2029; CHECK-NEXT: sqrdmulh v0.8h, v0.8h, v1.h[7]2030; CHECK-NEXT: ret2031entry:2032 %vqrdmulh2.i = tail call <8 x i16> @llvm.aarch64.neon.sqrdmulh.laneq.v8i16.v8i16(<8 x i16> %a, <8 x i16> %v, i32 7)2033 ret <8 x i16> %vqrdmulh2.i2034}2035 2036define <2 x i32> @test_vqrdmulh_lane_s32(<2 x i32> %a, <2 x i32> %v) {2037; CHECK-LABEL: test_vqrdmulh_lane_s32:2038; CHECK: // %bb.0: // %entry2039; CHECK-NEXT: // kill: def $d1 killed $d1 def $q12040; CHECK-NEXT: sqrdmulh v0.2s, v0.2s, v1.s[1]2041; CHECK-NEXT: ret2042entry:2043 %shuffle = shufflevector <2 x i32> %v, <2 x i32> undef, <2 x i32> <i32 1, i32 1>2044 %vqrdmulh2.i = tail call <2 x i32> @llvm.aarch64.neon.sqrdmulh.v2i32(<2 x i32> %a, <2 x i32> %shuffle)2045 ret <2 x i32> %vqrdmulh2.i2046}2047 2048define <2 x i32> @test_vqrdmulh_lane_s32_intrinsic(<2 x i32> %a, <2 x i32> %v) {2049; CHECK-LABEL: test_vqrdmulh_lane_s32_intrinsic:2050; CHECK: // %bb.0: // %entry2051; CHECK-NEXT: // kill: def $d1 killed $d1 def $q12052; CHECK-NEXT: sqrdmulh v0.2s, v0.2s, v1.s[1]2053; CHECK-NEXT: ret2054entry:2055 %vqrdmulh2.i = tail call <2 x i32> @llvm.aarch64.neon.sqrdmulh.lane.v2i32.v2i32(<2 x i32> %a, <2 x i32> %v, i32 1)2056 ret <2 x i32> %vqrdmulh2.i2057}2058 2059define <2 x i32> @test_vqrdmulh_laneq_s32_intrinsic_lo(<2 x i32> %a, <4 x i32> %v) {2060; CHECK-LABEL: test_vqrdmulh_laneq_s32_intrinsic_lo:2061; CHECK: // %bb.0: // %entry2062; CHECK-NEXT: sqrdmulh v0.2s, v0.2s, v1.s[1]2063; CHECK-NEXT: ret2064entry:2065 %vqrdmulh2.i = tail call <2 x i32> @llvm.aarch64.neon.sqrdmulh.laneq.v2i32.v4i32(<2 x i32> %a, <4 x i32> %v, i32 1)2066 ret <2 x i32> %vqrdmulh2.i2067}2068 2069define <2 x i32> @test_vqrdmulh_laneq_s32_intrinsic_hi(<2 x i32> %a, <4 x i32> %v) {2070; CHECK-LABEL: test_vqrdmulh_laneq_s32_intrinsic_hi:2071; CHECK: // %bb.0: // %entry2072; CHECK-NEXT: sqrdmulh v0.2s, v0.2s, v1.s[3]2073; CHECK-NEXT: ret2074entry:2075 %vqrdmulh2.i = tail call <2 x i32> @llvm.aarch64.neon.sqrdmulh.laneq.v2i32.v4i32(<2 x i32> %a, <4 x i32> %v, i32 3)2076 ret <2 x i32> %vqrdmulh2.i2077}2078 2079define <4 x i32> @test_vqrdmulhq_lane_s32(<4 x i32> %a, <2 x i32> %v) {2080; CHECK-LABEL: test_vqrdmulhq_lane_s32:2081; CHECK: // %bb.0: // %entry2082; CHECK-NEXT: // kill: def $d1 killed $d1 def $q12083; CHECK-NEXT: sqrdmulh v0.4s, v0.4s, v1.s[1]2084; CHECK-NEXT: ret2085entry:2086 %shuffle = shufflevector <2 x i32> %v, <2 x i32> undef, <4 x i32> <i32 1, i32 1, i32 1, i32 1>2087 %vqrdmulh2.i = tail call <4 x i32> @llvm.aarch64.neon.sqrdmulh.v4i32(<4 x i32> %a, <4 x i32> %shuffle)2088 ret <4 x i32> %vqrdmulh2.i2089}2090 2091define <4 x i32> @test_vqrdmulhq_lane_s32_intrinsic(<4 x i32> %a, <2 x i32> %v) {2092; CHECK-LABEL: test_vqrdmulhq_lane_s32_intrinsic:2093; CHECK: // %bb.0: // %entry2094; CHECK-NEXT: // kill: def $d1 killed $d1 def $q12095; CHECK-NEXT: sqrdmulh v0.4s, v0.4s, v1.s[1]2096; CHECK-NEXT: ret2097entry:2098 %vqrdmulh2.i = tail call <4 x i32> @llvm.aarch64.neon.sqrdmulh.lane.v4i32.v2i32(<4 x i32> %a, <2 x i32> %v, i32 1)2099 ret <4 x i32> %vqrdmulh2.i2100}2101 2102define <4 x i32> @test_vqrdmulhq_laneq_s32_intrinsic_lo(<4 x i32> %a, <4 x i32> %v) {2103; CHECK-LABEL: test_vqrdmulhq_laneq_s32_intrinsic_lo:2104; CHECK: // %bb.0: // %entry2105; CHECK-NEXT: sqrdmulh v0.4s, v0.4s, v1.s[1]2106; CHECK-NEXT: ret2107entry:2108 %vqrdmulh2.i = tail call <4 x i32> @llvm.aarch64.neon.sqrdmulh.laneq.v4i32.v4i32(<4 x i32> %a, <4 x i32> %v, i32 1)2109 ret <4 x i32> %vqrdmulh2.i2110}2111 2112define <4 x i32> @test_vqrdmulhq_laneq_s32_intrinsic_hi(<4 x i32> %a, <4 x i32> %v) {2113; CHECK-LABEL: test_vqrdmulhq_laneq_s32_intrinsic_hi:2114; CHECK: // %bb.0: // %entry2115; CHECK-NEXT: sqrdmulh v0.4s, v0.4s, v1.s[3]2116; CHECK-NEXT: ret2117entry:2118 %vqrdmulh2.i = tail call <4 x i32> @llvm.aarch64.neon.sqrdmulh.laneq.v4i32.v4i32(<4 x i32> %a, <4 x i32> %v, i32 3)2119 ret <4 x i32> %vqrdmulh2.i2120}2121 2122define <2 x float> @test_vmul_lane_f32(<2 x float> %a, <2 x float> %v) {2123; CHECK-LABEL: test_vmul_lane_f32:2124; CHECK: // %bb.0: // %entry2125; CHECK-NEXT: // kill: def $d1 killed $d1 def $q12126; CHECK-NEXT: fmul v0.2s, v0.2s, v1.s[1]2127; CHECK-NEXT: ret2128entry:2129 %shuffle = shufflevector <2 x float> %v, <2 x float> undef, <2 x i32> <i32 1, i32 1>2130 %mul = fmul <2 x float> %shuffle, %a2131 ret <2 x float> %mul2132}2133 2134define <1 x double> @test_vmul_lane_f64(<1 x double> %a, <1 x double> %v) {2135; CHECK-LABEL: test_vmul_lane_f64:2136; CHECK: // %bb.0: // %entry2137; CHECK-NEXT: fmul d0, d0, d12138; CHECK-NEXT: ret2139entry:2140 %0 = bitcast <1 x double> %a to <8 x i8>2141 %1 = bitcast <8 x i8> %0 to double2142 %extract = extractelement <1 x double> %v, i32 02143 %2 = fmul double %1, %extract2144 %3 = insertelement <1 x double> undef, double %2, i32 02145 ret <1 x double> %32146}2147 2148define <4 x float> @test_vmulq_lane_f32(<4 x float> %a, <2 x float> %v) {2149; CHECK-LABEL: test_vmulq_lane_f32:2150; CHECK: // %bb.0: // %entry2151; CHECK-NEXT: // kill: def $d1 killed $d1 def $q12152; CHECK-NEXT: fmul v0.4s, v0.4s, v1.s[1]2153; CHECK-NEXT: ret2154entry:2155 %shuffle = shufflevector <2 x float> %v, <2 x float> undef, <4 x i32> <i32 1, i32 1, i32 1, i32 1>2156 %mul = fmul <4 x float> %shuffle, %a2157 ret <4 x float> %mul2158}2159 2160define <2 x double> @test_vmulq_lane_f64(<2 x double> %a, <1 x double> %v) {2161; CHECK-LABEL: test_vmulq_lane_f64:2162; CHECK: // %bb.0: // %entry2163; CHECK-NEXT: // kill: def $d1 killed $d1 def $q12164; CHECK-NEXT: fmul v0.2d, v0.2d, v1.d[0]2165; CHECK-NEXT: ret2166entry:2167 %shuffle = shufflevector <1 x double> %v, <1 x double> undef, <2 x i32> zeroinitializer2168 %mul = fmul <2 x double> %shuffle, %a2169 ret <2 x double> %mul2170}2171 2172define <2 x float> @test_vmul_laneq_f32(<2 x float> %a, <4 x float> %v) {2173; CHECK-LABEL: test_vmul_laneq_f32:2174; CHECK: // %bb.0: // %entry2175; CHECK-NEXT: fmul v0.2s, v0.2s, v1.s[3]2176; CHECK-NEXT: ret2177entry:2178 %shuffle = shufflevector <4 x float> %v, <4 x float> undef, <2 x i32> <i32 3, i32 3>2179 %mul = fmul <2 x float> %shuffle, %a2180 ret <2 x float> %mul2181}2182 2183define <2 x float> @test_vmul_laneq3_f32_bitcast(<2 x float> %a, <2 x double> %v) {2184; CHECK-SD-LABEL: test_vmul_laneq3_f32_bitcast:2185; CHECK-SD: // %bb.0:2186; CHECK-SD-NEXT: fmul v0.2s, v0.2s, v1.s[3]2187; CHECK-SD-NEXT: ret2188;2189; CHECK-GI-LABEL: test_vmul_laneq3_f32_bitcast:2190; CHECK-GI: // %bb.0:2191; CHECK-GI-NEXT: mov d1, v1.d[1]2192; CHECK-GI-NEXT: fmul v0.2s, v0.2s, v1.s[1]2193; CHECK-GI-NEXT: ret2194 %extract = shufflevector <2 x double> %v, <2 x double> undef, <1 x i32> <i32 1>2195 %bc = bitcast <1 x double> %extract to <2 x float>2196 %splat = shufflevector <2 x float> %bc, <2 x float> undef, <2 x i32> <i32 1, i32 1>2197 %mul = fmul <2 x float> %splat, %a2198 ret <2 x float> %mul2199}2200 2201define <2 x float> @test_vmul_laneq2_f32_bitcast(<2 x float> %a, <2 x double> %v) {2202; CHECK-SD-LABEL: test_vmul_laneq2_f32_bitcast:2203; CHECK-SD: // %bb.0:2204; CHECK-SD-NEXT: fmul v0.2s, v0.2s, v1.s[2]2205; CHECK-SD-NEXT: ret2206;2207; CHECK-GI-LABEL: test_vmul_laneq2_f32_bitcast:2208; CHECK-GI: // %bb.0:2209; CHECK-GI-NEXT: mov d1, v1.d[1]2210; CHECK-GI-NEXT: fmul v0.2s, v0.2s, v1.s[0]2211; CHECK-GI-NEXT: ret2212 %extract = shufflevector <2 x double> %v, <2 x double> undef, <1 x i32> <i32 1>2213 %bc = bitcast <1 x double> %extract to <2 x float>2214 %splat = shufflevector <2 x float> %bc, <2 x float> undef, <2 x i32> <i32 0, i32 0>2215 %mul = fmul <2 x float> %splat, %a2216 ret <2 x float> %mul2217}2218 2219define <4 x i16> @test_vadd_laneq5_i16_bitcast(<4 x i16> %a, <2 x double> %v) {2220; CHECK-SD-LABEL: test_vadd_laneq5_i16_bitcast:2221; CHECK-SD: // %bb.0:2222; CHECK-SD-NEXT: dup v1.4h, v1.h[5]2223; CHECK-SD-NEXT: add v0.4h, v1.4h, v0.4h2224; CHECK-SD-NEXT: ret2225;2226; CHECK-GI-LABEL: test_vadd_laneq5_i16_bitcast:2227; CHECK-GI: // %bb.0:2228; CHECK-GI-NEXT: mov d1, v1.d[1]2229; CHECK-GI-NEXT: dup v1.4h, v1.h[1]2230; CHECK-GI-NEXT: add v0.4h, v1.4h, v0.4h2231; CHECK-GI-NEXT: ret2232 %extract = shufflevector <2 x double> %v, <2 x double> undef, <1 x i32> <i32 1>2233 %bc = bitcast <1 x double> %extract to <4 x i16>2234 %splat = shufflevector <4 x i16> %bc, <4 x i16> undef, <4 x i32> <i32 1, i32 1, i32 1, i32 1>2235 %r = add <4 x i16> %splat, %a2236 ret <4 x i16> %r2237}2238 2239; TODO: The pattern in LowerVECTOR_SHUFFLE does not match what we are looking for.2240 2241define <4 x i16> @test_vadd_lane2_i16_bitcast_bigger_aligned(<4 x i16> %a, <16 x i8> %v) {2242; CHECK-SD-LABEL: test_vadd_lane2_i16_bitcast_bigger_aligned:2243; CHECK-SD: // %bb.0:2244; CHECK-SD-NEXT: dup v1.4h, v1.h[2]2245; CHECK-SD-NEXT: dup v1.4h, v1.h[1]2246; CHECK-SD-NEXT: add v0.4h, v1.4h, v0.4h2247; CHECK-SD-NEXT: ret2248;2249; CHECK-GI-LABEL: test_vadd_lane2_i16_bitcast_bigger_aligned:2250; CHECK-GI: // %bb.0:2251; CHECK-GI-NEXT: ext v1.16b, v1.16b, v0.16b, #22252; CHECK-GI-NEXT: dup v1.4h, v1.h[1]2253; CHECK-GI-NEXT: add v0.4h, v1.4h, v0.4h2254; CHECK-GI-NEXT: ret2255 %extract = shufflevector <16 x i8> %v, <16 x i8> undef, <8 x i32> <i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9>2256 %bc = bitcast <8 x i8> %extract to <4 x i16>2257 %splat = shufflevector <4 x i16> %bc, <4 x i16> undef, <4 x i32> <i32 1, i32 1, i32 1, i32 1>2258 %r = add <4 x i16> %splat, %a2259 ret <4 x i16> %r2260}2261 2262define <4 x i16> @test_vadd_lane5_i16_bitcast_bigger_aligned(<4 x i16> %a, <16 x i8> %v) {2263; CHECK-SD-LABEL: test_vadd_lane5_i16_bitcast_bigger_aligned:2264; CHECK-SD: // %bb.0:2265; CHECK-SD-NEXT: dup v1.4h, v1.h[5]2266; CHECK-SD-NEXT: add v0.4h, v1.4h, v0.4h2267; CHECK-SD-NEXT: ret2268;2269; CHECK-GI-LABEL: test_vadd_lane5_i16_bitcast_bigger_aligned:2270; CHECK-GI: // %bb.0:2271; CHECK-GI-NEXT: mov d1, v1.d[1]2272; CHECK-GI-NEXT: dup v1.4h, v1.h[1]2273; CHECK-GI-NEXT: add v0.4h, v1.4h, v0.4h2274; CHECK-GI-NEXT: ret2275 %extract = shufflevector <16 x i8> %v, <16 x i8> undef, <8 x i32> <i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>2276 %bc = bitcast <8 x i8> %extract to <4 x i16>2277 %splat = shufflevector <4 x i16> %bc, <4 x i16> undef, <4 x i32> <i32 1, i32 1, i32 1, i32 1>2278 %r = add <4 x i16> %splat, %a2279 ret <4 x i16> %r2280}2281 2282; Negative test - can't dup bytes {3,4} of v8i16.2283 2284define <4 x i16> @test_vadd_lane_i16_bitcast_bigger_unaligned(<4 x i16> %a, <16 x i8> %v) {2285; CHECK-SD-LABEL: test_vadd_lane_i16_bitcast_bigger_unaligned:2286; CHECK-SD: // %bb.0:2287; CHECK-SD-NEXT: ext v1.8b, v1.8b, v0.8b, #12288; CHECK-SD-NEXT: dup v1.4h, v1.h[1]2289; CHECK-SD-NEXT: add v0.4h, v1.4h, v0.4h2290; CHECK-SD-NEXT: ret2291;2292; CHECK-GI-LABEL: test_vadd_lane_i16_bitcast_bigger_unaligned:2293; CHECK-GI: // %bb.0:2294; CHECK-GI-NEXT: ext v1.16b, v1.16b, v0.16b, #12295; CHECK-GI-NEXT: dup v1.4h, v1.h[1]2296; CHECK-GI-NEXT: add v0.4h, v1.4h, v0.4h2297; CHECK-GI-NEXT: ret2298 %extract = shufflevector <16 x i8> %v, <16 x i8> undef, <8 x i32> <i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8>2299 %bc = bitcast <8 x i8> %extract to <4 x i16>2300 %splat = shufflevector <4 x i16> %bc, <4 x i16> undef, <4 x i32> <i32 1, i32 1, i32 1, i32 1>2301 %r = add <4 x i16> %splat, %a2302 ret <4 x i16> %r2303}2304 2305define <1 x double> @test_vmul_laneq_f64(<1 x double> %a, <2 x double> %v) {2306; CHECK-LABEL: test_vmul_laneq_f64:2307; CHECK: // %bb.0: // %entry2308; CHECK-NEXT: fmul d0, d0, v1.d[1]2309; CHECK-NEXT: ret2310entry:2311 %0 = bitcast <1 x double> %a to <8 x i8>2312 %1 = bitcast <8 x i8> %0 to double2313 %extract = extractelement <2 x double> %v, i32 12314 %2 = fmul double %1, %extract2315 %3 = insertelement <1 x double> undef, double %2, i32 02316 ret <1 x double> %32317}2318 2319define <4 x float> @test_vmulq_laneq_f32(<4 x float> %a, <4 x float> %v) {2320; CHECK-LABEL: test_vmulq_laneq_f32:2321; CHECK: // %bb.0: // %entry2322; CHECK-NEXT: fmul v0.4s, v0.4s, v1.s[3]2323; CHECK-NEXT: ret2324entry:2325 %shuffle = shufflevector <4 x float> %v, <4 x float> undef, <4 x i32> <i32 3, i32 3, i32 3, i32 3>2326 %mul = fmul <4 x float> %shuffle, %a2327 ret <4 x float> %mul2328}2329 2330define <2 x double> @test_vmulq_laneq_f64(<2 x double> %a, <2 x double> %v) {2331; CHECK-LABEL: test_vmulq_laneq_f64:2332; CHECK: // %bb.0: // %entry2333; CHECK-NEXT: fmul v0.2d, v0.2d, v1.d[1]2334; CHECK-NEXT: ret2335entry:2336 %shuffle = shufflevector <2 x double> %v, <2 x double> undef, <2 x i32> <i32 1, i32 1>2337 %mul = fmul <2 x double> %shuffle, %a2338 ret <2 x double> %mul2339}2340 2341define <2 x float> @test_vmulx_lane_f32(<2 x float> %a, <2 x float> %v) {2342; CHECK-LABEL: test_vmulx_lane_f32:2343; CHECK: // %bb.0: // %entry2344; CHECK-NEXT: // kill: def $d1 killed $d1 def $q12345; CHECK-NEXT: fmulx v0.2s, v0.2s, v1.s[1]2346; CHECK-NEXT: ret2347entry:2348 %shuffle = shufflevector <2 x float> %v, <2 x float> undef, <2 x i32> <i32 1, i32 1>2349 %vmulx2.i = tail call <2 x float> @llvm.aarch64.neon.fmulx.v2f32(<2 x float> %a, <2 x float> %shuffle)2350 ret <2 x float> %vmulx2.i2351}2352 2353define <4 x float> @test_vmulxq_lane_f32(<4 x float> %a, <2 x float> %v) {2354; CHECK-LABEL: test_vmulxq_lane_f32:2355; CHECK: // %bb.0: // %entry2356; CHECK-NEXT: // kill: def $d1 killed $d1 def $q12357; CHECK-NEXT: fmulx v0.4s, v0.4s, v1.s[1]2358; CHECK-NEXT: ret2359entry:2360 %shuffle = shufflevector <2 x float> %v, <2 x float> undef, <4 x i32> <i32 1, i32 1, i32 1, i32 1>2361 %vmulx2.i = tail call <4 x float> @llvm.aarch64.neon.fmulx.v4f32(<4 x float> %a, <4 x float> %shuffle)2362 ret <4 x float> %vmulx2.i2363}2364 2365define <1 x double> @test_vmulx_lane_f64(<1 x double> %a, <1 x double> %v) {2366; CHECK-LABEL: test_vmulx_lane_f64:2367; CHECK: // %bb.0: // %entry2368; CHECK-NEXT: fmulx d0, d0, d12369; CHECK-NEXT: ret2370entry:2371 %vget_lane = extractelement <1 x double> %a, i64 02372 %vget_lane3 = extractelement <1 x double> %v, i64 02373 %vmulxd_f64.i = tail call double @llvm.aarch64.neon.fmulx.f64(double %vget_lane, double %vget_lane3)2374 %vset_lane = insertelement <1 x double> poison, double %vmulxd_f64.i, i64 02375 ret <1 x double> %vset_lane2376}2377 2378define <2 x double> @test_vmulxq_lane_f64(<2 x double> %a, <1 x double> %v) {2379; CHECK-SD-LABEL: test_vmulxq_lane_f64:2380; CHECK-SD: // %bb.0: // %entry2381; CHECK-SD-NEXT: // kill: def $d1 killed $d1 def $q12382; CHECK-SD-NEXT: fmulx v0.2d, v0.2d, v1.d[0]2383; CHECK-SD-NEXT: ret2384;2385; CHECK-GI-LABEL: test_vmulxq_lane_f64:2386; CHECK-GI: // %bb.0: // %entry2387; CHECK-GI-NEXT: // kill: def $d1 killed $d1 def $q12388; CHECK-GI-NEXT: dup v1.2d, v1.d[0]2389; CHECK-GI-NEXT: fmulx v0.2d, v0.2d, v1.2d2390; CHECK-GI-NEXT: ret2391entry:2392 %shuffle = shufflevector <1 x double> %v, <1 x double> undef, <2 x i32> zeroinitializer2393 %vmulx2.i = tail call <2 x double> @llvm.aarch64.neon.fmulx.v2f64(<2 x double> %a, <2 x double> %shuffle)2394 ret <2 x double> %vmulx2.i2395}2396 2397define <2 x float> @test_vmulx_laneq_f32(<2 x float> %a, <4 x float> %v) {2398; CHECK-LABEL: test_vmulx_laneq_f32:2399; CHECK: // %bb.0: // %entry2400; CHECK-NEXT: fmulx v0.2s, v0.2s, v1.s[3]2401; CHECK-NEXT: ret2402entry:2403 %shuffle = shufflevector <4 x float> %v, <4 x float> undef, <2 x i32> <i32 3, i32 3>2404 %vmulx2.i = tail call <2 x float> @llvm.aarch64.neon.fmulx.v2f32(<2 x float> %a, <2 x float> %shuffle)2405 ret <2 x float> %vmulx2.i2406}2407 2408define <4 x float> @test_vmulxq_laneq_f32(<4 x float> %a, <4 x float> %v) {2409; CHECK-LABEL: test_vmulxq_laneq_f32:2410; CHECK: // %bb.0: // %entry2411; CHECK-NEXT: fmulx v0.4s, v0.4s, v1.s[3]2412; CHECK-NEXT: ret2413entry:2414 %shuffle = shufflevector <4 x float> %v, <4 x float> undef, <4 x i32> <i32 3, i32 3, i32 3, i32 3>2415 %vmulx2.i = tail call <4 x float> @llvm.aarch64.neon.fmulx.v4f32(<4 x float> %a, <4 x float> %shuffle)2416 ret <4 x float> %vmulx2.i2417}2418 2419define <1 x double> @test_vmulx_laneq_f64(<1 x double> %a, <2 x double> %v) {2420; CHECK-LABEL: test_vmulx_laneq_f64:2421; CHECK: // %bb.0: // %entry2422; CHECK-NEXT: fmulx d0, d0, v1.d[1]2423; CHECK-NEXT: ret2424entry:2425 %vget_lane = extractelement <1 x double> %a, i64 02426 %vgetq_lane = extractelement <2 x double> %v, i64 12427 %vmulxd_f64.i = tail call double @llvm.aarch64.neon.fmulx.f64(double %vget_lane, double %vgetq_lane)2428 %vset_lane = insertelement <1 x double> poison, double %vmulxd_f64.i, i64 02429 ret <1 x double> %vset_lane2430}2431 2432define <2 x double> @test_vmulxq_laneq_f64(<2 x double> %a, <2 x double> %v) {2433; CHECK-LABEL: test_vmulxq_laneq_f64:2434; CHECK: // %bb.0: // %entry2435; CHECK-NEXT: fmulx v0.2d, v0.2d, v1.d[1]2436; CHECK-NEXT: ret2437entry:2438 %shuffle = shufflevector <2 x double> %v, <2 x double> undef, <2 x i32> <i32 1, i32 1>2439 %vmulx2.i = tail call <2 x double> @llvm.aarch64.neon.fmulx.v2f64(<2 x double> %a, <2 x double> %shuffle)2440 ret <2 x double> %vmulx2.i2441}2442 2443define <4 x i16> @test_vmla_lane_s16_0(<4 x i16> %a, <4 x i16> %b, <4 x i16> %v) {2444; CHECK-LABEL: test_vmla_lane_s16_0:2445; CHECK: // %bb.0: // %entry2446; CHECK-NEXT: // kill: def $d2 killed $d2 def $q22447; CHECK-NEXT: mla v0.4h, v1.4h, v2.h[0]2448; CHECK-NEXT: ret2449entry:2450 %shuffle = shufflevector <4 x i16> %v, <4 x i16> undef, <4 x i32> zeroinitializer2451 %mul = mul <4 x i16> %shuffle, %b2452 %add = add <4 x i16> %mul, %a2453 ret <4 x i16> %add2454}2455 2456define <8 x i16> @test_vmlaq_lane_s16_0(<8 x i16> %a, <8 x i16> %b, <4 x i16> %v) {2457; CHECK-LABEL: test_vmlaq_lane_s16_0:2458; CHECK: // %bb.0: // %entry2459; CHECK-NEXT: // kill: def $d2 killed $d2 def $q22460; CHECK-NEXT: mla v0.8h, v1.8h, v2.h[0]2461; CHECK-NEXT: ret2462entry:2463 %shuffle = shufflevector <4 x i16> %v, <4 x i16> undef, <8 x i32> zeroinitializer2464 %mul = mul <8 x i16> %shuffle, %b2465 %add = add <8 x i16> %mul, %a2466 ret <8 x i16> %add2467}2468 2469define <2 x i32> @test_vmla_lane_s32_0(<2 x i32> %a, <2 x i32> %b, <2 x i32> %v) {2470; CHECK-LABEL: test_vmla_lane_s32_0:2471; CHECK: // %bb.0: // %entry2472; CHECK-NEXT: // kill: def $d2 killed $d2 def $q22473; CHECK-NEXT: mla v0.2s, v1.2s, v2.s[0]2474; CHECK-NEXT: ret2475entry:2476 %shuffle = shufflevector <2 x i32> %v, <2 x i32> undef, <2 x i32> zeroinitializer2477 %mul = mul <2 x i32> %shuffle, %b2478 %add = add <2 x i32> %mul, %a2479 ret <2 x i32> %add2480}2481 2482define <4 x i32> @test_vmlaq_lane_s32_0(<4 x i32> %a, <4 x i32> %b, <2 x i32> %v) {2483; CHECK-LABEL: test_vmlaq_lane_s32_0:2484; CHECK: // %bb.0: // %entry2485; CHECK-NEXT: // kill: def $d2 killed $d2 def $q22486; CHECK-NEXT: mla v0.4s, v1.4s, v2.s[0]2487; CHECK-NEXT: ret2488entry:2489 %shuffle = shufflevector <2 x i32> %v, <2 x i32> undef, <4 x i32> zeroinitializer2490 %mul = mul <4 x i32> %shuffle, %b2491 %add = add <4 x i32> %mul, %a2492 ret <4 x i32> %add2493}2494 2495define <4 x i16> @test_vmla_laneq_s16_0(<4 x i16> %a, <4 x i16> %b, <8 x i16> %v) {2496; CHECK-LABEL: test_vmla_laneq_s16_0:2497; CHECK: // %bb.0: // %entry2498; CHECK-NEXT: mla v0.4h, v1.4h, v2.h[0]2499; CHECK-NEXT: ret2500entry:2501 %shuffle = shufflevector <8 x i16> %v, <8 x i16> undef, <4 x i32> zeroinitializer2502 %mul = mul <4 x i16> %shuffle, %b2503 %add = add <4 x i16> %mul, %a2504 ret <4 x i16> %add2505}2506 2507define <8 x i16> @test_vmlaq_laneq_s16_0(<8 x i16> %a, <8 x i16> %b, <8 x i16> %v) {2508; CHECK-LABEL: test_vmlaq_laneq_s16_0:2509; CHECK: // %bb.0: // %entry2510; CHECK-NEXT: mla v0.8h, v1.8h, v2.h[0]2511; CHECK-NEXT: ret2512entry:2513 %shuffle = shufflevector <8 x i16> %v, <8 x i16> undef, <8 x i32> zeroinitializer2514 %mul = mul <8 x i16> %shuffle, %b2515 %add = add <8 x i16> %mul, %a2516 ret <8 x i16> %add2517}2518 2519define <2 x i32> @test_vmla_laneq_s32_0(<2 x i32> %a, <2 x i32> %b, <4 x i32> %v) {2520; CHECK-LABEL: test_vmla_laneq_s32_0:2521; CHECK: // %bb.0: // %entry2522; CHECK-NEXT: mla v0.2s, v1.2s, v2.s[0]2523; CHECK-NEXT: ret2524entry:2525 %shuffle = shufflevector <4 x i32> %v, <4 x i32> undef, <2 x i32> zeroinitializer2526 %mul = mul <2 x i32> %shuffle, %b2527 %add = add <2 x i32> %mul, %a2528 ret <2 x i32> %add2529}2530 2531define <4 x i32> @test_vmlaq_laneq_s32_0(<4 x i32> %a, <4 x i32> %b, <4 x i32> %v) {2532; CHECK-LABEL: test_vmlaq_laneq_s32_0:2533; CHECK: // %bb.0: // %entry2534; CHECK-NEXT: mla v0.4s, v1.4s, v2.s[0]2535; CHECK-NEXT: ret2536entry:2537 %shuffle = shufflevector <4 x i32> %v, <4 x i32> undef, <4 x i32> zeroinitializer2538 %mul = mul <4 x i32> %shuffle, %b2539 %add = add <4 x i32> %mul, %a2540 ret <4 x i32> %add2541}2542 2543define <4 x i16> @test_vmls_lane_s16_0(<4 x i16> %a, <4 x i16> %b, <4 x i16> %v) {2544; CHECK-LABEL: test_vmls_lane_s16_0:2545; CHECK: // %bb.0: // %entry2546; CHECK-NEXT: // kill: def $d2 killed $d2 def $q22547; CHECK-NEXT: mls v0.4h, v1.4h, v2.h[0]2548; CHECK-NEXT: ret2549entry:2550 %shuffle = shufflevector <4 x i16> %v, <4 x i16> undef, <4 x i32> zeroinitializer2551 %mul = mul <4 x i16> %shuffle, %b2552 %sub = sub <4 x i16> %a, %mul2553 ret <4 x i16> %sub2554}2555 2556define <8 x i16> @test_vmlsq_lane_s16_0(<8 x i16> %a, <8 x i16> %b, <4 x i16> %v) {2557; CHECK-LABEL: test_vmlsq_lane_s16_0:2558; CHECK: // %bb.0: // %entry2559; CHECK-NEXT: // kill: def $d2 killed $d2 def $q22560; CHECK-NEXT: mls v0.8h, v1.8h, v2.h[0]2561; CHECK-NEXT: ret2562entry:2563 %shuffle = shufflevector <4 x i16> %v, <4 x i16> undef, <8 x i32> zeroinitializer2564 %mul = mul <8 x i16> %shuffle, %b2565 %sub = sub <8 x i16> %a, %mul2566 ret <8 x i16> %sub2567}2568 2569define <2 x i32> @test_vmls_lane_s32_0(<2 x i32> %a, <2 x i32> %b, <2 x i32> %v) {2570; CHECK-LABEL: test_vmls_lane_s32_0:2571; CHECK: // %bb.0: // %entry2572; CHECK-NEXT: // kill: def $d2 killed $d2 def $q22573; CHECK-NEXT: mls v0.2s, v1.2s, v2.s[0]2574; CHECK-NEXT: ret2575entry:2576 %shuffle = shufflevector <2 x i32> %v, <2 x i32> undef, <2 x i32> zeroinitializer2577 %mul = mul <2 x i32> %shuffle, %b2578 %sub = sub <2 x i32> %a, %mul2579 ret <2 x i32> %sub2580}2581 2582define <4 x i32> @test_vmlsq_lane_s32_0(<4 x i32> %a, <4 x i32> %b, <2 x i32> %v) {2583; CHECK-LABEL: test_vmlsq_lane_s32_0:2584; CHECK: // %bb.0: // %entry2585; CHECK-NEXT: // kill: def $d2 killed $d2 def $q22586; CHECK-NEXT: mls v0.4s, v1.4s, v2.s[0]2587; CHECK-NEXT: ret2588entry:2589 %shuffle = shufflevector <2 x i32> %v, <2 x i32> undef, <4 x i32> zeroinitializer2590 %mul = mul <4 x i32> %shuffle, %b2591 %sub = sub <4 x i32> %a, %mul2592 ret <4 x i32> %sub2593}2594 2595define <4 x i16> @test_vmls_laneq_s16_0(<4 x i16> %a, <4 x i16> %b, <8 x i16> %v) {2596; CHECK-LABEL: test_vmls_laneq_s16_0:2597; CHECK: // %bb.0: // %entry2598; CHECK-NEXT: mls v0.4h, v1.4h, v2.h[0]2599; CHECK-NEXT: ret2600entry:2601 %shuffle = shufflevector <8 x i16> %v, <8 x i16> undef, <4 x i32> zeroinitializer2602 %mul = mul <4 x i16> %shuffle, %b2603 %sub = sub <4 x i16> %a, %mul2604 ret <4 x i16> %sub2605}2606 2607define <8 x i16> @test_vmlsq_laneq_s16_0(<8 x i16> %a, <8 x i16> %b, <8 x i16> %v) {2608; CHECK-LABEL: test_vmlsq_laneq_s16_0:2609; CHECK: // %bb.0: // %entry2610; CHECK-NEXT: mls v0.8h, v1.8h, v2.h[0]2611; CHECK-NEXT: ret2612entry:2613 %shuffle = shufflevector <8 x i16> %v, <8 x i16> undef, <8 x i32> zeroinitializer2614 %mul = mul <8 x i16> %shuffle, %b2615 %sub = sub <8 x i16> %a, %mul2616 ret <8 x i16> %sub2617}2618 2619define <2 x i32> @test_vmls_laneq_s32_0(<2 x i32> %a, <2 x i32> %b, <4 x i32> %v) {2620; CHECK-LABEL: test_vmls_laneq_s32_0:2621; CHECK: // %bb.0: // %entry2622; CHECK-NEXT: mls v0.2s, v1.2s, v2.s[0]2623; CHECK-NEXT: ret2624entry:2625 %shuffle = shufflevector <4 x i32> %v, <4 x i32> undef, <2 x i32> zeroinitializer2626 %mul = mul <2 x i32> %shuffle, %b2627 %sub = sub <2 x i32> %a, %mul2628 ret <2 x i32> %sub2629}2630 2631define <4 x i32> @test_vmlsq_laneq_s32_0(<4 x i32> %a, <4 x i32> %b, <4 x i32> %v) {2632; CHECK-LABEL: test_vmlsq_laneq_s32_0:2633; CHECK: // %bb.0: // %entry2634; CHECK-NEXT: mls v0.4s, v1.4s, v2.s[0]2635; CHECK-NEXT: ret2636entry:2637 %shuffle = shufflevector <4 x i32> %v, <4 x i32> undef, <4 x i32> zeroinitializer2638 %mul = mul <4 x i32> %shuffle, %b2639 %sub = sub <4 x i32> %a, %mul2640 ret <4 x i32> %sub2641}2642 2643define <4 x i16> @test_vmul_lane_s16_0(<4 x i16> %a, <4 x i16> %v) {2644; CHECK-LABEL: test_vmul_lane_s16_0:2645; CHECK: // %bb.0: // %entry2646; CHECK-NEXT: // kill: def $d1 killed $d1 def $q12647; CHECK-NEXT: mul v0.4h, v0.4h, v1.h[0]2648; CHECK-NEXT: ret2649entry:2650 %shuffle = shufflevector <4 x i16> %v, <4 x i16> undef, <4 x i32> zeroinitializer2651 %mul = mul <4 x i16> %shuffle, %a2652 ret <4 x i16> %mul2653}2654 2655define <8 x i16> @test_vmulq_lane_s16_0(<8 x i16> %a, <4 x i16> %v) {2656; CHECK-LABEL: test_vmulq_lane_s16_0:2657; CHECK: // %bb.0: // %entry2658; CHECK-NEXT: // kill: def $d1 killed $d1 def $q12659; CHECK-NEXT: mul v0.8h, v0.8h, v1.h[0]2660; CHECK-NEXT: ret2661entry:2662 %shuffle = shufflevector <4 x i16> %v, <4 x i16> undef, <8 x i32> zeroinitializer2663 %mul = mul <8 x i16> %shuffle, %a2664 ret <8 x i16> %mul2665}2666 2667define <2 x i32> @test_vmul_lane_s32_0(<2 x i32> %a, <2 x i32> %v) {2668; CHECK-LABEL: test_vmul_lane_s32_0:2669; CHECK: // %bb.0: // %entry2670; CHECK-NEXT: // kill: def $d1 killed $d1 def $q12671; CHECK-NEXT: mul v0.2s, v0.2s, v1.s[0]2672; CHECK-NEXT: ret2673entry:2674 %shuffle = shufflevector <2 x i32> %v, <2 x i32> undef, <2 x i32> zeroinitializer2675 %mul = mul <2 x i32> %shuffle, %a2676 ret <2 x i32> %mul2677}2678 2679define <4 x i32> @test_vmulq_lane_s32_0(<4 x i32> %a, <2 x i32> %v) {2680; CHECK-LABEL: test_vmulq_lane_s32_0:2681; CHECK: // %bb.0: // %entry2682; CHECK-NEXT: // kill: def $d1 killed $d1 def $q12683; CHECK-NEXT: mul v0.4s, v0.4s, v1.s[0]2684; CHECK-NEXT: ret2685entry:2686 %shuffle = shufflevector <2 x i32> %v, <2 x i32> undef, <4 x i32> zeroinitializer2687 %mul = mul <4 x i32> %shuffle, %a2688 ret <4 x i32> %mul2689}2690 2691define <4 x i16> @test_vmul_lane_u16_0(<4 x i16> %a, <4 x i16> %v) {2692; CHECK-LABEL: test_vmul_lane_u16_0:2693; CHECK: // %bb.0: // %entry2694; CHECK-NEXT: // kill: def $d1 killed $d1 def $q12695; CHECK-NEXT: mul v0.4h, v0.4h, v1.h[0]2696; CHECK-NEXT: ret2697entry:2698 %shuffle = shufflevector <4 x i16> %v, <4 x i16> undef, <4 x i32> zeroinitializer2699 %mul = mul <4 x i16> %shuffle, %a2700 ret <4 x i16> %mul2701}2702 2703define <8 x i16> @test_vmulq_lane_u16_0(<8 x i16> %a, <4 x i16> %v) {2704; CHECK-LABEL: test_vmulq_lane_u16_0:2705; CHECK: // %bb.0: // %entry2706; CHECK-NEXT: // kill: def $d1 killed $d1 def $q12707; CHECK-NEXT: mul v0.8h, v0.8h, v1.h[0]2708; CHECK-NEXT: ret2709entry:2710 %shuffle = shufflevector <4 x i16> %v, <4 x i16> undef, <8 x i32> zeroinitializer2711 %mul = mul <8 x i16> %shuffle, %a2712 ret <8 x i16> %mul2713}2714 2715define <2 x i32> @test_vmul_lane_u32_0(<2 x i32> %a, <2 x i32> %v) {2716; CHECK-LABEL: test_vmul_lane_u32_0:2717; CHECK: // %bb.0: // %entry2718; CHECK-NEXT: // kill: def $d1 killed $d1 def $q12719; CHECK-NEXT: mul v0.2s, v0.2s, v1.s[0]2720; CHECK-NEXT: ret2721entry:2722 %shuffle = shufflevector <2 x i32> %v, <2 x i32> undef, <2 x i32> zeroinitializer2723 %mul = mul <2 x i32> %shuffle, %a2724 ret <2 x i32> %mul2725}2726 2727define <4 x i32> @test_vmulq_lane_u32_0(<4 x i32> %a, <2 x i32> %v) {2728; CHECK-LABEL: test_vmulq_lane_u32_0:2729; CHECK: // %bb.0: // %entry2730; CHECK-NEXT: // kill: def $d1 killed $d1 def $q12731; CHECK-NEXT: mul v0.4s, v0.4s, v1.s[0]2732; CHECK-NEXT: ret2733entry:2734 %shuffle = shufflevector <2 x i32> %v, <2 x i32> undef, <4 x i32> zeroinitializer2735 %mul = mul <4 x i32> %shuffle, %a2736 ret <4 x i32> %mul2737}2738 2739define <4 x i16> @test_vmul_laneq_s16_0(<4 x i16> %a, <8 x i16> %v) {2740; CHECK-LABEL: test_vmul_laneq_s16_0:2741; CHECK: // %bb.0: // %entry2742; CHECK-NEXT: mul v0.4h, v0.4h, v1.h[0]2743; CHECK-NEXT: ret2744entry:2745 %shuffle = shufflevector <8 x i16> %v, <8 x i16> undef, <4 x i32> zeroinitializer2746 %mul = mul <4 x i16> %shuffle, %a2747 ret <4 x i16> %mul2748}2749 2750define <8 x i16> @test_vmulq_laneq_s16_0(<8 x i16> %a, <8 x i16> %v) {2751; CHECK-LABEL: test_vmulq_laneq_s16_0:2752; CHECK: // %bb.0: // %entry2753; CHECK-NEXT: mul v0.8h, v0.8h, v1.h[0]2754; CHECK-NEXT: ret2755entry:2756 %shuffle = shufflevector <8 x i16> %v, <8 x i16> undef, <8 x i32> zeroinitializer2757 %mul = mul <8 x i16> %shuffle, %a2758 ret <8 x i16> %mul2759}2760 2761define <2 x i32> @test_vmul_laneq_s32_0(<2 x i32> %a, <4 x i32> %v) {2762; CHECK-LABEL: test_vmul_laneq_s32_0:2763; CHECK: // %bb.0: // %entry2764; CHECK-NEXT: mul v0.2s, v0.2s, v1.s[0]2765; CHECK-NEXT: ret2766entry:2767 %shuffle = shufflevector <4 x i32> %v, <4 x i32> undef, <2 x i32> zeroinitializer2768 %mul = mul <2 x i32> %shuffle, %a2769 ret <2 x i32> %mul2770}2771 2772define <4 x i32> @test_vmulq_laneq_s32_0(<4 x i32> %a, <4 x i32> %v) {2773; CHECK-LABEL: test_vmulq_laneq_s32_0:2774; CHECK: // %bb.0: // %entry2775; CHECK-NEXT: mul v0.4s, v0.4s, v1.s[0]2776; CHECK-NEXT: ret2777entry:2778 %shuffle = shufflevector <4 x i32> %v, <4 x i32> undef, <4 x i32> zeroinitializer2779 %mul = mul <4 x i32> %shuffle, %a2780 ret <4 x i32> %mul2781}2782 2783define <4 x i16> @test_vmul_laneq_u16_0(<4 x i16> %a, <8 x i16> %v) {2784; CHECK-LABEL: test_vmul_laneq_u16_0:2785; CHECK: // %bb.0: // %entry2786; CHECK-NEXT: mul v0.4h, v0.4h, v1.h[0]2787; CHECK-NEXT: ret2788entry:2789 %shuffle = shufflevector <8 x i16> %v, <8 x i16> undef, <4 x i32> zeroinitializer2790 %mul = mul <4 x i16> %shuffle, %a2791 ret <4 x i16> %mul2792}2793 2794define <8 x i16> @test_vmulq_laneq_u16_0(<8 x i16> %a, <8 x i16> %v) {2795; CHECK-LABEL: test_vmulq_laneq_u16_0:2796; CHECK: // %bb.0: // %entry2797; CHECK-NEXT: mul v0.8h, v0.8h, v1.h[0]2798; CHECK-NEXT: ret2799entry:2800 %shuffle = shufflevector <8 x i16> %v, <8 x i16> undef, <8 x i32> zeroinitializer2801 %mul = mul <8 x i16> %shuffle, %a2802 ret <8 x i16> %mul2803}2804 2805define <2 x i32> @test_vmul_laneq_u32_0(<2 x i32> %a, <4 x i32> %v) {2806; CHECK-LABEL: test_vmul_laneq_u32_0:2807; CHECK: // %bb.0: // %entry2808; CHECK-NEXT: mul v0.2s, v0.2s, v1.s[0]2809; CHECK-NEXT: ret2810entry:2811 %shuffle = shufflevector <4 x i32> %v, <4 x i32> undef, <2 x i32> zeroinitializer2812 %mul = mul <2 x i32> %shuffle, %a2813 ret <2 x i32> %mul2814}2815 2816define <4 x i32> @test_vmulq_laneq_u32_0(<4 x i32> %a, <4 x i32> %v) {2817; CHECK-LABEL: test_vmulq_laneq_u32_0:2818; CHECK: // %bb.0: // %entry2819; CHECK-NEXT: mul v0.4s, v0.4s, v1.s[0]2820; CHECK-NEXT: ret2821entry:2822 %shuffle = shufflevector <4 x i32> %v, <4 x i32> undef, <4 x i32> zeroinitializer2823 %mul = mul <4 x i32> %shuffle, %a2824 ret <4 x i32> %mul2825}2826 2827define <2 x float> @test_vfma_lane_f32_0(<2 x float> %a, <2 x float> %b, <2 x float> %v) {2828; CHECK-LABEL: test_vfma_lane_f32_0:2829; CHECK: // %bb.0: // %entry2830; CHECK-NEXT: // kill: def $d2 killed $d2 def $q22831; CHECK-NEXT: fmla v0.2s, v1.2s, v2.s[0]2832; CHECK-NEXT: ret2833entry:2834 %lane = shufflevector <2 x float> %v, <2 x float> undef, <2 x i32> zeroinitializer2835 %0 = tail call <2 x float> @llvm.fma.v2f32(<2 x float> %lane, <2 x float> %b, <2 x float> %a)2836 ret <2 x float> %02837}2838 2839define <4 x float> @test_vfmaq_lane_f32_0(<4 x float> %a, <4 x float> %b, <2 x float> %v) {2840; CHECK-LABEL: test_vfmaq_lane_f32_0:2841; CHECK: // %bb.0: // %entry2842; CHECK-NEXT: // kill: def $d2 killed $d2 def $q22843; CHECK-NEXT: fmla v0.4s, v1.4s, v2.s[0]2844; CHECK-NEXT: ret2845entry:2846 %lane = shufflevector <2 x float> %v, <2 x float> undef, <4 x i32> zeroinitializer2847 %0 = tail call <4 x float> @llvm.fma.v4f32(<4 x float> %lane, <4 x float> %b, <4 x float> %a)2848 ret <4 x float> %02849}2850 2851define <2 x float> @test_vfma_laneq_f32_0(<2 x float> %a, <2 x float> %b, <4 x float> %v) {2852; CHECK-LABEL: test_vfma_laneq_f32_0:2853; CHECK: // %bb.0: // %entry2854; CHECK-NEXT: fmla v0.2s, v1.2s, v2.s[0]2855; CHECK-NEXT: ret2856entry:2857 %lane = shufflevector <4 x float> %v, <4 x float> undef, <2 x i32> zeroinitializer2858 %0 = tail call <2 x float> @llvm.fma.v2f32(<2 x float> %lane, <2 x float> %b, <2 x float> %a)2859 ret <2 x float> %02860}2861 2862define <4 x float> @test_vfmaq_laneq_f32_0(<4 x float> %a, <4 x float> %b, <4 x float> %v) {2863; CHECK-LABEL: test_vfmaq_laneq_f32_0:2864; CHECK: // %bb.0: // %entry2865; CHECK-NEXT: fmla v0.4s, v1.4s, v2.s[0]2866; CHECK-NEXT: ret2867entry:2868 %lane = shufflevector <4 x float> %v, <4 x float> undef, <4 x i32> zeroinitializer2869 %0 = tail call <4 x float> @llvm.fma.v4f32(<4 x float> %lane, <4 x float> %b, <4 x float> %a)2870 ret <4 x float> %02871}2872 2873define <2 x float> @test_vfms_lane_f32_0(<2 x float> %a, <2 x float> %b, <2 x float> %v) {2874; CHECK-LABEL: test_vfms_lane_f32_0:2875; CHECK: // %bb.0: // %entry2876; CHECK-NEXT: // kill: def $d2 killed $d2 def $q22877; CHECK-NEXT: fmls v0.2s, v1.2s, v2.s[0]2878; CHECK-NEXT: ret2879entry:2880 %sub = fsub <2 x float> <float -0.000000e+00, float -0.000000e+00>, %v2881 %lane = shufflevector <2 x float> %sub, <2 x float> undef, <2 x i32> zeroinitializer2882 %0 = tail call <2 x float> @llvm.fma.v2f32(<2 x float> %lane, <2 x float> %b, <2 x float> %a)2883 ret <2 x float> %02884}2885 2886define <4 x float> @test_vfmsq_lane_f32_0(<4 x float> %a, <4 x float> %b, <2 x float> %v) {2887; CHECK-LABEL: test_vfmsq_lane_f32_0:2888; CHECK: // %bb.0: // %entry2889; CHECK-NEXT: // kill: def $d2 killed $d2 def $q22890; CHECK-NEXT: fmls v0.4s, v1.4s, v2.s[0]2891; CHECK-NEXT: ret2892entry:2893 %sub = fsub <2 x float> <float -0.000000e+00, float -0.000000e+00>, %v2894 %lane = shufflevector <2 x float> %sub, <2 x float> undef, <4 x i32> zeroinitializer2895 %0 = tail call <4 x float> @llvm.fma.v4f32(<4 x float> %lane, <4 x float> %b, <4 x float> %a)2896 ret <4 x float> %02897}2898 2899define <2 x float> @test_vfms_laneq_f32_0(<2 x float> %a, <2 x float> %b, <4 x float> %v) {2900; CHECK-LABEL: test_vfms_laneq_f32_0:2901; CHECK: // %bb.0: // %entry2902; CHECK-NEXT: fmls v0.2s, v1.2s, v2.s[0]2903; CHECK-NEXT: ret2904entry:2905 %sub = fsub <4 x float> <float -0.000000e+00, float -0.000000e+00, float -0.000000e+00, float -0.000000e+00>, %v2906 %lane = shufflevector <4 x float> %sub, <4 x float> undef, <2 x i32> zeroinitializer2907 %0 = tail call <2 x float> @llvm.fma.v2f32(<2 x float> %lane, <2 x float> %b, <2 x float> %a)2908 ret <2 x float> %02909}2910 2911define <4 x float> @test_vfmsq_laneq_f32_0(<4 x float> %a, <4 x float> %b, <4 x float> %v) {2912; CHECK-LABEL: test_vfmsq_laneq_f32_0:2913; CHECK: // %bb.0: // %entry2914; CHECK-NEXT: fmls v0.4s, v1.4s, v2.s[0]2915; CHECK-NEXT: ret2916entry:2917 %sub = fsub <4 x float> <float -0.000000e+00, float -0.000000e+00, float -0.000000e+00, float -0.000000e+00>, %v2918 %lane = shufflevector <4 x float> %sub, <4 x float> undef, <4 x i32> zeroinitializer2919 %0 = tail call <4 x float> @llvm.fma.v4f32(<4 x float> %lane, <4 x float> %b, <4 x float> %a)2920 ret <4 x float> %02921}2922 2923define <2 x double> @test_vfmaq_laneq_f64_0(<2 x double> %a, <2 x double> %b, <2 x double> %v) {2924; CHECK-LABEL: test_vfmaq_laneq_f64_0:2925; CHECK: // %bb.0: // %entry2926; CHECK-NEXT: fmla v0.2d, v1.2d, v2.d[0]2927; CHECK-NEXT: ret2928entry:2929 %lane = shufflevector <2 x double> %v, <2 x double> undef, <2 x i32> zeroinitializer2930 %0 = tail call <2 x double> @llvm.fma.v2f64(<2 x double> %lane, <2 x double> %b, <2 x double> %a)2931 ret <2 x double> %02932}2933 2934define <2 x double> @test_vfmsq_laneq_f64_0(<2 x double> %a, <2 x double> %b, <2 x double> %v) {2935; CHECK-LABEL: test_vfmsq_laneq_f64_0:2936; CHECK: // %bb.0: // %entry2937; CHECK-NEXT: fmls v0.2d, v1.2d, v2.d[0]2938; CHECK-NEXT: ret2939entry:2940 %sub = fsub <2 x double> <double -0.000000e+00, double -0.000000e+00>, %v2941 %lane = shufflevector <2 x double> %sub, <2 x double> undef, <2 x i32> zeroinitializer2942 %0 = tail call <2 x double> @llvm.fma.v2f64(<2 x double> %lane, <2 x double> %b, <2 x double> %a)2943 ret <2 x double> %02944}2945 2946define <4 x i32> @test_vmlal_lane_s16_0(<4 x i32> %a, <4 x i16> %b, <4 x i16> %v) {2947; CHECK-LABEL: test_vmlal_lane_s16_0:2948; CHECK: // %bb.0: // %entry2949; CHECK-NEXT: // kill: def $d2 killed $d2 def $q22950; CHECK-NEXT: smlal v0.4s, v1.4h, v2.h[0]2951; CHECK-NEXT: ret2952entry:2953 %shuffle = shufflevector <4 x i16> %v, <4 x i16> undef, <4 x i32> zeroinitializer2954 %vmull2.i = tail call <4 x i32> @llvm.aarch64.neon.smull.v4i32(<4 x i16> %b, <4 x i16> %shuffle)2955 %add = add <4 x i32> %vmull2.i, %a2956 ret <4 x i32> %add2957}2958 2959define <2 x i64> @test_vmlal_lane_s32_0(<2 x i64> %a, <2 x i32> %b, <2 x i32> %v) {2960; CHECK-LABEL: test_vmlal_lane_s32_0:2961; CHECK: // %bb.0: // %entry2962; CHECK-NEXT: // kill: def $d2 killed $d2 def $q22963; CHECK-NEXT: smlal v0.2d, v1.2s, v2.s[0]2964; CHECK-NEXT: ret2965entry:2966 %shuffle = shufflevector <2 x i32> %v, <2 x i32> undef, <2 x i32> zeroinitializer2967 %vmull2.i = tail call <2 x i64> @llvm.aarch64.neon.smull.v2i64(<2 x i32> %b, <2 x i32> %shuffle)2968 %add = add <2 x i64> %vmull2.i, %a2969 ret <2 x i64> %add2970}2971 2972define <4 x i32> @test_vmlal_laneq_s16_0(<4 x i32> %a, <4 x i16> %b, <8 x i16> %v) {2973; CHECK-LABEL: test_vmlal_laneq_s16_0:2974; CHECK: // %bb.0: // %entry2975; CHECK-NEXT: smlal v0.4s, v1.4h, v2.h[0]2976; CHECK-NEXT: ret2977entry:2978 %shuffle = shufflevector <8 x i16> %v, <8 x i16> undef, <4 x i32> zeroinitializer2979 %vmull2.i = tail call <4 x i32> @llvm.aarch64.neon.smull.v4i32(<4 x i16> %b, <4 x i16> %shuffle)2980 %add = add <4 x i32> %vmull2.i, %a2981 ret <4 x i32> %add2982}2983 2984define <2 x i64> @test_vmlal_laneq_s32_0(<2 x i64> %a, <2 x i32> %b, <4 x i32> %v) {2985; CHECK-LABEL: test_vmlal_laneq_s32_0:2986; CHECK: // %bb.0: // %entry2987; CHECK-NEXT: smlal v0.2d, v1.2s, v2.s[0]2988; CHECK-NEXT: ret2989entry:2990 %shuffle = shufflevector <4 x i32> %v, <4 x i32> undef, <2 x i32> zeroinitializer2991 %vmull2.i = tail call <2 x i64> @llvm.aarch64.neon.smull.v2i64(<2 x i32> %b, <2 x i32> %shuffle)2992 %add = add <2 x i64> %vmull2.i, %a2993 ret <2 x i64> %add2994}2995 2996define <4 x i32> @test_vmlal_high_lane_s16_0(<4 x i32> %a, <8 x i16> %b, <4 x i16> %v) {2997; CHECK-SD-LABEL: test_vmlal_high_lane_s16_0:2998; CHECK-SD: // %bb.0: // %entry2999; CHECK-SD-NEXT: // kill: def $d2 killed $d2 def $q23000; CHECK-SD-NEXT: smlal2 v0.4s, v1.8h, v2.h[0]3001; CHECK-SD-NEXT: ret3002;3003; CHECK-GI-LABEL: test_vmlal_high_lane_s16_0:3004; CHECK-GI: // %bb.0: // %entry3005; CHECK-GI-NEXT: mov d1, v1.d[1]3006; CHECK-GI-NEXT: // kill: def $d2 killed $d2 def $q23007; CHECK-GI-NEXT: smlal v0.4s, v1.4h, v2.h[0]3008; CHECK-GI-NEXT: ret3009entry:3010 %shuffle.i = shufflevector <8 x i16> %b, <8 x i16> undef, <4 x i32> <i32 4, i32 5, i32 6, i32 7>3011 %shuffle = shufflevector <4 x i16> %v, <4 x i16> undef, <4 x i32> zeroinitializer3012 %vmull2.i = tail call <4 x i32> @llvm.aarch64.neon.smull.v4i32(<4 x i16> %shuffle.i, <4 x i16> %shuffle)3013 %add = add <4 x i32> %vmull2.i, %a3014 ret <4 x i32> %add3015}3016 3017define <2 x i64> @test_vmlal_high_lane_s32_0(<2 x i64> %a, <4 x i32> %b, <2 x i32> %v) {3018; CHECK-SD-LABEL: test_vmlal_high_lane_s32_0:3019; CHECK-SD: // %bb.0: // %entry3020; CHECK-SD-NEXT: // kill: def $d2 killed $d2 def $q23021; CHECK-SD-NEXT: smlal2 v0.2d, v1.4s, v2.s[0]3022; CHECK-SD-NEXT: ret3023;3024; CHECK-GI-LABEL: test_vmlal_high_lane_s32_0:3025; CHECK-GI: // %bb.0: // %entry3026; CHECK-GI-NEXT: mov d1, v1.d[1]3027; CHECK-GI-NEXT: // kill: def $d2 killed $d2 def $q23028; CHECK-GI-NEXT: smlal v0.2d, v1.2s, v2.s[0]3029; CHECK-GI-NEXT: ret3030entry:3031 %shuffle.i = shufflevector <4 x i32> %b, <4 x i32> undef, <2 x i32> <i32 2, i32 3>3032 %shuffle = shufflevector <2 x i32> %v, <2 x i32> undef, <2 x i32> zeroinitializer3033 %vmull2.i = tail call <2 x i64> @llvm.aarch64.neon.smull.v2i64(<2 x i32> %shuffle.i, <2 x i32> %shuffle)3034 %add = add <2 x i64> %vmull2.i, %a3035 ret <2 x i64> %add3036}3037 3038define <4 x i32> @test_vmlal_high_laneq_s16_0(<4 x i32> %a, <8 x i16> %b, <8 x i16> %v) {3039; CHECK-SD-LABEL: test_vmlal_high_laneq_s16_0:3040; CHECK-SD: // %bb.0: // %entry3041; CHECK-SD-NEXT: smlal2 v0.4s, v1.8h, v2.h[0]3042; CHECK-SD-NEXT: ret3043;3044; CHECK-GI-LABEL: test_vmlal_high_laneq_s16_0:3045; CHECK-GI: // %bb.0: // %entry3046; CHECK-GI-NEXT: mov d1, v1.d[1]3047; CHECK-GI-NEXT: smlal v0.4s, v1.4h, v2.h[0]3048; CHECK-GI-NEXT: ret3049entry:3050 %shuffle.i = shufflevector <8 x i16> %b, <8 x i16> undef, <4 x i32> <i32 4, i32 5, i32 6, i32 7>3051 %shuffle = shufflevector <8 x i16> %v, <8 x i16> undef, <4 x i32> zeroinitializer3052 %vmull2.i = tail call <4 x i32> @llvm.aarch64.neon.smull.v4i32(<4 x i16> %shuffle.i, <4 x i16> %shuffle)3053 %add = add <4 x i32> %vmull2.i, %a3054 ret <4 x i32> %add3055}3056 3057define <2 x i64> @test_vmlal_high_laneq_s32_0(<2 x i64> %a, <4 x i32> %b, <4 x i32> %v) {3058; CHECK-SD-LABEL: test_vmlal_high_laneq_s32_0:3059; CHECK-SD: // %bb.0: // %entry3060; CHECK-SD-NEXT: smlal2 v0.2d, v1.4s, v2.s[0]3061; CHECK-SD-NEXT: ret3062;3063; CHECK-GI-LABEL: test_vmlal_high_laneq_s32_0:3064; CHECK-GI: // %bb.0: // %entry3065; CHECK-GI-NEXT: mov d1, v1.d[1]3066; CHECK-GI-NEXT: smlal v0.2d, v1.2s, v2.s[0]3067; CHECK-GI-NEXT: ret3068entry:3069 %shuffle.i = shufflevector <4 x i32> %b, <4 x i32> undef, <2 x i32> <i32 2, i32 3>3070 %shuffle = shufflevector <4 x i32> %v, <4 x i32> undef, <2 x i32> zeroinitializer3071 %vmull2.i = tail call <2 x i64> @llvm.aarch64.neon.smull.v2i64(<2 x i32> %shuffle.i, <2 x i32> %shuffle)3072 %add = add <2 x i64> %vmull2.i, %a3073 ret <2 x i64> %add3074}3075 3076define <4 x i32> @test_vmlsl_lane_s16_0(<4 x i32> %a, <4 x i16> %b, <4 x i16> %v) {3077; CHECK-LABEL: test_vmlsl_lane_s16_0:3078; CHECK: // %bb.0: // %entry3079; CHECK-NEXT: // kill: def $d2 killed $d2 def $q23080; CHECK-NEXT: smlsl v0.4s, v1.4h, v2.h[0]3081; CHECK-NEXT: ret3082entry:3083 %shuffle = shufflevector <4 x i16> %v, <4 x i16> undef, <4 x i32> zeroinitializer3084 %vmull2.i = tail call <4 x i32> @llvm.aarch64.neon.smull.v4i32(<4 x i16> %b, <4 x i16> %shuffle)3085 %sub = sub <4 x i32> %a, %vmull2.i3086 ret <4 x i32> %sub3087}3088 3089define <2 x i64> @test_vmlsl_lane_s32_0(<2 x i64> %a, <2 x i32> %b, <2 x i32> %v) {3090; CHECK-LABEL: test_vmlsl_lane_s32_0:3091; CHECK: // %bb.0: // %entry3092; CHECK-NEXT: // kill: def $d2 killed $d2 def $q23093; CHECK-NEXT: smlsl v0.2d, v1.2s, v2.s[0]3094; CHECK-NEXT: ret3095entry:3096 %shuffle = shufflevector <2 x i32> %v, <2 x i32> undef, <2 x i32> zeroinitializer3097 %vmull2.i = tail call <2 x i64> @llvm.aarch64.neon.smull.v2i64(<2 x i32> %b, <2 x i32> %shuffle)3098 %sub = sub <2 x i64> %a, %vmull2.i3099 ret <2 x i64> %sub3100}3101 3102define <4 x i32> @test_vmlsl_laneq_s16_0(<4 x i32> %a, <4 x i16> %b, <8 x i16> %v) {3103; CHECK-LABEL: test_vmlsl_laneq_s16_0:3104; CHECK: // %bb.0: // %entry3105; CHECK-NEXT: smlsl v0.4s, v1.4h, v2.h[0]3106; CHECK-NEXT: ret3107entry:3108 %shuffle = shufflevector <8 x i16> %v, <8 x i16> undef, <4 x i32> zeroinitializer3109 %vmull2.i = tail call <4 x i32> @llvm.aarch64.neon.smull.v4i32(<4 x i16> %b, <4 x i16> %shuffle)3110 %sub = sub <4 x i32> %a, %vmull2.i3111 ret <4 x i32> %sub3112}3113 3114define <2 x i64> @test_vmlsl_laneq_s32_0(<2 x i64> %a, <2 x i32> %b, <4 x i32> %v) {3115; CHECK-LABEL: test_vmlsl_laneq_s32_0:3116; CHECK: // %bb.0: // %entry3117; CHECK-NEXT: smlsl v0.2d, v1.2s, v2.s[0]3118; CHECK-NEXT: ret3119entry:3120 %shuffle = shufflevector <4 x i32> %v, <4 x i32> undef, <2 x i32> zeroinitializer3121 %vmull2.i = tail call <2 x i64> @llvm.aarch64.neon.smull.v2i64(<2 x i32> %b, <2 x i32> %shuffle)3122 %sub = sub <2 x i64> %a, %vmull2.i3123 ret <2 x i64> %sub3124}3125 3126define <4 x i32> @test_vmlsl_high_lane_s16_0(<4 x i32> %a, <8 x i16> %b, <4 x i16> %v) {3127; CHECK-SD-LABEL: test_vmlsl_high_lane_s16_0:3128; CHECK-SD: // %bb.0: // %entry3129; CHECK-SD-NEXT: // kill: def $d2 killed $d2 def $q23130; CHECK-SD-NEXT: smlsl2 v0.4s, v1.8h, v2.h[0]3131; CHECK-SD-NEXT: ret3132;3133; CHECK-GI-LABEL: test_vmlsl_high_lane_s16_0:3134; CHECK-GI: // %bb.0: // %entry3135; CHECK-GI-NEXT: mov d1, v1.d[1]3136; CHECK-GI-NEXT: // kill: def $d2 killed $d2 def $q23137; CHECK-GI-NEXT: smlsl v0.4s, v1.4h, v2.h[0]3138; CHECK-GI-NEXT: ret3139entry:3140 %shuffle.i = shufflevector <8 x i16> %b, <8 x i16> undef, <4 x i32> <i32 4, i32 5, i32 6, i32 7>3141 %shuffle = shufflevector <4 x i16> %v, <4 x i16> undef, <4 x i32> zeroinitializer3142 %vmull2.i = tail call <4 x i32> @llvm.aarch64.neon.smull.v4i32(<4 x i16> %shuffle.i, <4 x i16> %shuffle)3143 %sub = sub <4 x i32> %a, %vmull2.i3144 ret <4 x i32> %sub3145}3146 3147define <2 x i64> @test_vmlsl_high_lane_s32_0(<2 x i64> %a, <4 x i32> %b, <2 x i32> %v) {3148; CHECK-SD-LABEL: test_vmlsl_high_lane_s32_0:3149; CHECK-SD: // %bb.0: // %entry3150; CHECK-SD-NEXT: // kill: def $d2 killed $d2 def $q23151; CHECK-SD-NEXT: smlsl2 v0.2d, v1.4s, v2.s[0]3152; CHECK-SD-NEXT: ret3153;3154; CHECK-GI-LABEL: test_vmlsl_high_lane_s32_0:3155; CHECK-GI: // %bb.0: // %entry3156; CHECK-GI-NEXT: mov d1, v1.d[1]3157; CHECK-GI-NEXT: // kill: def $d2 killed $d2 def $q23158; CHECK-GI-NEXT: smlsl v0.2d, v1.2s, v2.s[0]3159; CHECK-GI-NEXT: ret3160entry:3161 %shuffle.i = shufflevector <4 x i32> %b, <4 x i32> undef, <2 x i32> <i32 2, i32 3>3162 %shuffle = shufflevector <2 x i32> %v, <2 x i32> undef, <2 x i32> zeroinitializer3163 %vmull2.i = tail call <2 x i64> @llvm.aarch64.neon.smull.v2i64(<2 x i32> %shuffle.i, <2 x i32> %shuffle)3164 %sub = sub <2 x i64> %a, %vmull2.i3165 ret <2 x i64> %sub3166}3167 3168define <4 x i32> @test_vmlsl_high_laneq_s16_0(<4 x i32> %a, <8 x i16> %b, <8 x i16> %v) {3169; CHECK-SD-LABEL: test_vmlsl_high_laneq_s16_0:3170; CHECK-SD: // %bb.0: // %entry3171; CHECK-SD-NEXT: smlsl2 v0.4s, v1.8h, v2.h[0]3172; CHECK-SD-NEXT: ret3173;3174; CHECK-GI-LABEL: test_vmlsl_high_laneq_s16_0:3175; CHECK-GI: // %bb.0: // %entry3176; CHECK-GI-NEXT: mov d1, v1.d[1]3177; CHECK-GI-NEXT: smlsl v0.4s, v1.4h, v2.h[0]3178; CHECK-GI-NEXT: ret3179entry:3180 %shuffle.i = shufflevector <8 x i16> %b, <8 x i16> undef, <4 x i32> <i32 4, i32 5, i32 6, i32 7>3181 %shuffle = shufflevector <8 x i16> %v, <8 x i16> undef, <4 x i32> zeroinitializer3182 %vmull2.i = tail call <4 x i32> @llvm.aarch64.neon.smull.v4i32(<4 x i16> %shuffle.i, <4 x i16> %shuffle)3183 %sub = sub <4 x i32> %a, %vmull2.i3184 ret <4 x i32> %sub3185}3186 3187define <2 x i64> @test_vmlsl_high_laneq_s32_0(<2 x i64> %a, <4 x i32> %b, <4 x i32> %v) {3188; CHECK-SD-LABEL: test_vmlsl_high_laneq_s32_0:3189; CHECK-SD: // %bb.0: // %entry3190; CHECK-SD-NEXT: smlsl2 v0.2d, v1.4s, v2.s[0]3191; CHECK-SD-NEXT: ret3192;3193; CHECK-GI-LABEL: test_vmlsl_high_laneq_s32_0:3194; CHECK-GI: // %bb.0: // %entry3195; CHECK-GI-NEXT: mov d1, v1.d[1]3196; CHECK-GI-NEXT: smlsl v0.2d, v1.2s, v2.s[0]3197; CHECK-GI-NEXT: ret3198entry:3199 %shuffle.i = shufflevector <4 x i32> %b, <4 x i32> undef, <2 x i32> <i32 2, i32 3>3200 %shuffle = shufflevector <4 x i32> %v, <4 x i32> undef, <2 x i32> zeroinitializer3201 %vmull2.i = tail call <2 x i64> @llvm.aarch64.neon.smull.v2i64(<2 x i32> %shuffle.i, <2 x i32> %shuffle)3202 %sub = sub <2 x i64> %a, %vmull2.i3203 ret <2 x i64> %sub3204}3205 3206define <4 x i32> @test_vmlal_lane_u16_0(<4 x i32> %a, <4 x i16> %b, <4 x i16> %v) {3207; CHECK-LABEL: test_vmlal_lane_u16_0:3208; CHECK: // %bb.0: // %entry3209; CHECK-NEXT: // kill: def $d2 killed $d2 def $q23210; CHECK-NEXT: umlal v0.4s, v1.4h, v2.h[0]3211; CHECK-NEXT: ret3212entry:3213 %shuffle = shufflevector <4 x i16> %v, <4 x i16> undef, <4 x i32> zeroinitializer3214 %vmull2.i = tail call <4 x i32> @llvm.aarch64.neon.umull.v4i32(<4 x i16> %b, <4 x i16> %shuffle)3215 %add = add <4 x i32> %vmull2.i, %a3216 ret <4 x i32> %add3217}3218 3219define <2 x i64> @test_vmlal_lane_u32_0(<2 x i64> %a, <2 x i32> %b, <2 x i32> %v) {3220; CHECK-LABEL: test_vmlal_lane_u32_0:3221; CHECK: // %bb.0: // %entry3222; CHECK-NEXT: // kill: def $d2 killed $d2 def $q23223; CHECK-NEXT: umlal v0.2d, v1.2s, v2.s[0]3224; CHECK-NEXT: ret3225entry:3226 %shuffle = shufflevector <2 x i32> %v, <2 x i32> undef, <2 x i32> zeroinitializer3227 %vmull2.i = tail call <2 x i64> @llvm.aarch64.neon.umull.v2i64(<2 x i32> %b, <2 x i32> %shuffle)3228 %add = add <2 x i64> %vmull2.i, %a3229 ret <2 x i64> %add3230}3231 3232define <4 x i32> @test_vmlal_laneq_u16_0(<4 x i32> %a, <4 x i16> %b, <8 x i16> %v) {3233; CHECK-LABEL: test_vmlal_laneq_u16_0:3234; CHECK: // %bb.0: // %entry3235; CHECK-NEXT: umlal v0.4s, v1.4h, v2.h[0]3236; CHECK-NEXT: ret3237entry:3238 %shuffle = shufflevector <8 x i16> %v, <8 x i16> undef, <4 x i32> zeroinitializer3239 %vmull2.i = tail call <4 x i32> @llvm.aarch64.neon.umull.v4i32(<4 x i16> %b, <4 x i16> %shuffle)3240 %add = add <4 x i32> %vmull2.i, %a3241 ret <4 x i32> %add3242}3243 3244define <2 x i64> @test_vmlal_laneq_u32_0(<2 x i64> %a, <2 x i32> %b, <4 x i32> %v) {3245; CHECK-LABEL: test_vmlal_laneq_u32_0:3246; CHECK: // %bb.0: // %entry3247; CHECK-NEXT: umlal v0.2d, v1.2s, v2.s[0]3248; CHECK-NEXT: ret3249entry:3250 %shuffle = shufflevector <4 x i32> %v, <4 x i32> undef, <2 x i32> zeroinitializer3251 %vmull2.i = tail call <2 x i64> @llvm.aarch64.neon.umull.v2i64(<2 x i32> %b, <2 x i32> %shuffle)3252 %add = add <2 x i64> %vmull2.i, %a3253 ret <2 x i64> %add3254}3255 3256define <4 x i32> @test_vmlal_high_lane_u16_0(<4 x i32> %a, <8 x i16> %b, <4 x i16> %v) {3257; CHECK-SD-LABEL: test_vmlal_high_lane_u16_0:3258; CHECK-SD: // %bb.0: // %entry3259; CHECK-SD-NEXT: // kill: def $d2 killed $d2 def $q23260; CHECK-SD-NEXT: umlal2 v0.4s, v1.8h, v2.h[0]3261; CHECK-SD-NEXT: ret3262;3263; CHECK-GI-LABEL: test_vmlal_high_lane_u16_0:3264; CHECK-GI: // %bb.0: // %entry3265; CHECK-GI-NEXT: mov d1, v1.d[1]3266; CHECK-GI-NEXT: // kill: def $d2 killed $d2 def $q23267; CHECK-GI-NEXT: umlal v0.4s, v1.4h, v2.h[0]3268; CHECK-GI-NEXT: ret3269entry:3270 %shuffle.i = shufflevector <8 x i16> %b, <8 x i16> undef, <4 x i32> <i32 4, i32 5, i32 6, i32 7>3271 %shuffle = shufflevector <4 x i16> %v, <4 x i16> undef, <4 x i32> zeroinitializer3272 %vmull2.i = tail call <4 x i32> @llvm.aarch64.neon.umull.v4i32(<4 x i16> %shuffle.i, <4 x i16> %shuffle)3273 %add = add <4 x i32> %vmull2.i, %a3274 ret <4 x i32> %add3275}3276 3277define <2 x i64> @test_vmlal_high_lane_u32_0(<2 x i64> %a, <4 x i32> %b, <2 x i32> %v) {3278; CHECK-SD-LABEL: test_vmlal_high_lane_u32_0:3279; CHECK-SD: // %bb.0: // %entry3280; CHECK-SD-NEXT: // kill: def $d2 killed $d2 def $q23281; CHECK-SD-NEXT: umlal2 v0.2d, v1.4s, v2.s[0]3282; CHECK-SD-NEXT: ret3283;3284; CHECK-GI-LABEL: test_vmlal_high_lane_u32_0:3285; CHECK-GI: // %bb.0: // %entry3286; CHECK-GI-NEXT: mov d1, v1.d[1]3287; CHECK-GI-NEXT: // kill: def $d2 killed $d2 def $q23288; CHECK-GI-NEXT: umlal v0.2d, v1.2s, v2.s[0]3289; CHECK-GI-NEXT: ret3290entry:3291 %shuffle.i = shufflevector <4 x i32> %b, <4 x i32> undef, <2 x i32> <i32 2, i32 3>3292 %shuffle = shufflevector <2 x i32> %v, <2 x i32> undef, <2 x i32> zeroinitializer3293 %vmull2.i = tail call <2 x i64> @llvm.aarch64.neon.umull.v2i64(<2 x i32> %shuffle.i, <2 x i32> %shuffle)3294 %add = add <2 x i64> %vmull2.i, %a3295 ret <2 x i64> %add3296}3297 3298define <4 x i32> @test_vmlal_high_laneq_u16_0(<4 x i32> %a, <8 x i16> %b, <8 x i16> %v) {3299; CHECK-SD-LABEL: test_vmlal_high_laneq_u16_0:3300; CHECK-SD: // %bb.0: // %entry3301; CHECK-SD-NEXT: umlal2 v0.4s, v1.8h, v2.h[0]3302; CHECK-SD-NEXT: ret3303;3304; CHECK-GI-LABEL: test_vmlal_high_laneq_u16_0:3305; CHECK-GI: // %bb.0: // %entry3306; CHECK-GI-NEXT: mov d1, v1.d[1]3307; CHECK-GI-NEXT: umlal v0.4s, v1.4h, v2.h[0]3308; CHECK-GI-NEXT: ret3309entry:3310 %shuffle.i = shufflevector <8 x i16> %b, <8 x i16> undef, <4 x i32> <i32 4, i32 5, i32 6, i32 7>3311 %shuffle = shufflevector <8 x i16> %v, <8 x i16> undef, <4 x i32> zeroinitializer3312 %vmull2.i = tail call <4 x i32> @llvm.aarch64.neon.umull.v4i32(<4 x i16> %shuffle.i, <4 x i16> %shuffle)3313 %add = add <4 x i32> %vmull2.i, %a3314 ret <4 x i32> %add3315}3316 3317define <2 x i64> @test_vmlal_high_laneq_u32_0(<2 x i64> %a, <4 x i32> %b, <4 x i32> %v) {3318; CHECK-SD-LABEL: test_vmlal_high_laneq_u32_0:3319; CHECK-SD: // %bb.0: // %entry3320; CHECK-SD-NEXT: umlal2 v0.2d, v1.4s, v2.s[0]3321; CHECK-SD-NEXT: ret3322;3323; CHECK-GI-LABEL: test_vmlal_high_laneq_u32_0:3324; CHECK-GI: // %bb.0: // %entry3325; CHECK-GI-NEXT: mov d1, v1.d[1]3326; CHECK-GI-NEXT: umlal v0.2d, v1.2s, v2.s[0]3327; CHECK-GI-NEXT: ret3328entry:3329 %shuffle.i = shufflevector <4 x i32> %b, <4 x i32> undef, <2 x i32> <i32 2, i32 3>3330 %shuffle = shufflevector <4 x i32> %v, <4 x i32> undef, <2 x i32> zeroinitializer3331 %vmull2.i = tail call <2 x i64> @llvm.aarch64.neon.umull.v2i64(<2 x i32> %shuffle.i, <2 x i32> %shuffle)3332 %add = add <2 x i64> %vmull2.i, %a3333 ret <2 x i64> %add3334}3335 3336define <4 x i32> @test_vmlsl_lane_u16_0(<4 x i32> %a, <4 x i16> %b, <4 x i16> %v) {3337; CHECK-LABEL: test_vmlsl_lane_u16_0:3338; CHECK: // %bb.0: // %entry3339; CHECK-NEXT: // kill: def $d2 killed $d2 def $q23340; CHECK-NEXT: umlsl v0.4s, v1.4h, v2.h[0]3341; CHECK-NEXT: ret3342entry:3343 %shuffle = shufflevector <4 x i16> %v, <4 x i16> undef, <4 x i32> zeroinitializer3344 %vmull2.i = tail call <4 x i32> @llvm.aarch64.neon.umull.v4i32(<4 x i16> %b, <4 x i16> %shuffle)3345 %sub = sub <4 x i32> %a, %vmull2.i3346 ret <4 x i32> %sub3347}3348 3349define <2 x i64> @test_vmlsl_lane_u32_0(<2 x i64> %a, <2 x i32> %b, <2 x i32> %v) {3350; CHECK-LABEL: test_vmlsl_lane_u32_0:3351; CHECK: // %bb.0: // %entry3352; CHECK-NEXT: // kill: def $d2 killed $d2 def $q23353; CHECK-NEXT: umlsl v0.2d, v1.2s, v2.s[0]3354; CHECK-NEXT: ret3355entry:3356 %shuffle = shufflevector <2 x i32> %v, <2 x i32> undef, <2 x i32> zeroinitializer3357 %vmull2.i = tail call <2 x i64> @llvm.aarch64.neon.umull.v2i64(<2 x i32> %b, <2 x i32> %shuffle)3358 %sub = sub <2 x i64> %a, %vmull2.i3359 ret <2 x i64> %sub3360}3361 3362define <4 x i32> @test_vmlsl_laneq_u16_0(<4 x i32> %a, <4 x i16> %b, <8 x i16> %v) {3363; CHECK-LABEL: test_vmlsl_laneq_u16_0:3364; CHECK: // %bb.0: // %entry3365; CHECK-NEXT: umlsl v0.4s, v1.4h, v2.h[0]3366; CHECK-NEXT: ret3367entry:3368 %shuffle = shufflevector <8 x i16> %v, <8 x i16> undef, <4 x i32> zeroinitializer3369 %vmull2.i = tail call <4 x i32> @llvm.aarch64.neon.umull.v4i32(<4 x i16> %b, <4 x i16> %shuffle)3370 %sub = sub <4 x i32> %a, %vmull2.i3371 ret <4 x i32> %sub3372}3373 3374define <2 x i64> @test_vmlsl_laneq_u32_0(<2 x i64> %a, <2 x i32> %b, <4 x i32> %v) {3375; CHECK-LABEL: test_vmlsl_laneq_u32_0:3376; CHECK: // %bb.0: // %entry3377; CHECK-NEXT: umlsl v0.2d, v1.2s, v2.s[0]3378; CHECK-NEXT: ret3379entry:3380 %shuffle = shufflevector <4 x i32> %v, <4 x i32> undef, <2 x i32> zeroinitializer3381 %vmull2.i = tail call <2 x i64> @llvm.aarch64.neon.umull.v2i64(<2 x i32> %b, <2 x i32> %shuffle)3382 %sub = sub <2 x i64> %a, %vmull2.i3383 ret <2 x i64> %sub3384}3385 3386define <4 x i32> @test_vmlsl_high_lane_u16_0(<4 x i32> %a, <8 x i16> %b, <4 x i16> %v) {3387; CHECK-SD-LABEL: test_vmlsl_high_lane_u16_0:3388; CHECK-SD: // %bb.0: // %entry3389; CHECK-SD-NEXT: // kill: def $d2 killed $d2 def $q23390; CHECK-SD-NEXT: umlsl2 v0.4s, v1.8h, v2.h[0]3391; CHECK-SD-NEXT: ret3392;3393; CHECK-GI-LABEL: test_vmlsl_high_lane_u16_0:3394; CHECK-GI: // %bb.0: // %entry3395; CHECK-GI-NEXT: mov d1, v1.d[1]3396; CHECK-GI-NEXT: // kill: def $d2 killed $d2 def $q23397; CHECK-GI-NEXT: umlsl v0.4s, v1.4h, v2.h[0]3398; CHECK-GI-NEXT: ret3399entry:3400 %shuffle.i = shufflevector <8 x i16> %b, <8 x i16> undef, <4 x i32> <i32 4, i32 5, i32 6, i32 7>3401 %shuffle = shufflevector <4 x i16> %v, <4 x i16> undef, <4 x i32> zeroinitializer3402 %vmull2.i = tail call <4 x i32> @llvm.aarch64.neon.umull.v4i32(<4 x i16> %shuffle.i, <4 x i16> %shuffle)3403 %sub = sub <4 x i32> %a, %vmull2.i3404 ret <4 x i32> %sub3405}3406 3407define <2 x i64> @test_vmlsl_high_lane_u32_0(<2 x i64> %a, <4 x i32> %b, <2 x i32> %v) {3408; CHECK-SD-LABEL: test_vmlsl_high_lane_u32_0:3409; CHECK-SD: // %bb.0: // %entry3410; CHECK-SD-NEXT: // kill: def $d2 killed $d2 def $q23411; CHECK-SD-NEXT: umlsl2 v0.2d, v1.4s, v2.s[0]3412; CHECK-SD-NEXT: ret3413;3414; CHECK-GI-LABEL: test_vmlsl_high_lane_u32_0:3415; CHECK-GI: // %bb.0: // %entry3416; CHECK-GI-NEXT: mov d1, v1.d[1]3417; CHECK-GI-NEXT: // kill: def $d2 killed $d2 def $q23418; CHECK-GI-NEXT: umlsl v0.2d, v1.2s, v2.s[0]3419; CHECK-GI-NEXT: ret3420entry:3421 %shuffle.i = shufflevector <4 x i32> %b, <4 x i32> undef, <2 x i32> <i32 2, i32 3>3422 %shuffle = shufflevector <2 x i32> %v, <2 x i32> undef, <2 x i32> zeroinitializer3423 %vmull2.i = tail call <2 x i64> @llvm.aarch64.neon.umull.v2i64(<2 x i32> %shuffle.i, <2 x i32> %shuffle)3424 %sub = sub <2 x i64> %a, %vmull2.i3425 ret <2 x i64> %sub3426}3427 3428define <4 x i32> @test_vmlsl_high_laneq_u16_0(<4 x i32> %a, <8 x i16> %b, <8 x i16> %v) {3429; CHECK-SD-LABEL: test_vmlsl_high_laneq_u16_0:3430; CHECK-SD: // %bb.0: // %entry3431; CHECK-SD-NEXT: umlsl2 v0.4s, v1.8h, v2.h[0]3432; CHECK-SD-NEXT: ret3433;3434; CHECK-GI-LABEL: test_vmlsl_high_laneq_u16_0:3435; CHECK-GI: // %bb.0: // %entry3436; CHECK-GI-NEXT: mov d1, v1.d[1]3437; CHECK-GI-NEXT: umlsl v0.4s, v1.4h, v2.h[0]3438; CHECK-GI-NEXT: ret3439entry:3440 %shuffle.i = shufflevector <8 x i16> %b, <8 x i16> undef, <4 x i32> <i32 4, i32 5, i32 6, i32 7>3441 %shuffle = shufflevector <8 x i16> %v, <8 x i16> undef, <4 x i32> zeroinitializer3442 %vmull2.i = tail call <4 x i32> @llvm.aarch64.neon.umull.v4i32(<4 x i16> %shuffle.i, <4 x i16> %shuffle)3443 %sub = sub <4 x i32> %a, %vmull2.i3444 ret <4 x i32> %sub3445}3446 3447define <2 x i64> @test_vmlsl_high_laneq_u32_0(<2 x i64> %a, <4 x i32> %b, <4 x i32> %v) {3448; CHECK-SD-LABEL: test_vmlsl_high_laneq_u32_0:3449; CHECK-SD: // %bb.0: // %entry3450; CHECK-SD-NEXT: umlsl2 v0.2d, v1.4s, v2.s[0]3451; CHECK-SD-NEXT: ret3452;3453; CHECK-GI-LABEL: test_vmlsl_high_laneq_u32_0:3454; CHECK-GI: // %bb.0: // %entry3455; CHECK-GI-NEXT: mov d1, v1.d[1]3456; CHECK-GI-NEXT: umlsl v0.2d, v1.2s, v2.s[0]3457; CHECK-GI-NEXT: ret3458entry:3459 %shuffle.i = shufflevector <4 x i32> %b, <4 x i32> undef, <2 x i32> <i32 2, i32 3>3460 %shuffle = shufflevector <4 x i32> %v, <4 x i32> undef, <2 x i32> zeroinitializer3461 %vmull2.i = tail call <2 x i64> @llvm.aarch64.neon.umull.v2i64(<2 x i32> %shuffle.i, <2 x i32> %shuffle)3462 %sub = sub <2 x i64> %a, %vmull2.i3463 ret <2 x i64> %sub3464}3465 3466define <4 x i32> @test_vmull_lane_s16_0(<4 x i16> %a, <4 x i16> %v) {3467; CHECK-LABEL: test_vmull_lane_s16_0:3468; CHECK: // %bb.0: // %entry3469; CHECK-NEXT: // kill: def $d1 killed $d1 def $q13470; CHECK-NEXT: smull v0.4s, v0.4h, v1.h[0]3471; CHECK-NEXT: ret3472entry:3473 %shuffle = shufflevector <4 x i16> %v, <4 x i16> undef, <4 x i32> zeroinitializer3474 %vmull2.i = tail call <4 x i32> @llvm.aarch64.neon.smull.v4i32(<4 x i16> %a, <4 x i16> %shuffle)3475 ret <4 x i32> %vmull2.i3476}3477 3478define <2 x i64> @test_vmull_lane_s32_0(<2 x i32> %a, <2 x i32> %v) {3479; CHECK-LABEL: test_vmull_lane_s32_0:3480; CHECK: // %bb.0: // %entry3481; CHECK-NEXT: // kill: def $d1 killed $d1 def $q13482; CHECK-NEXT: smull v0.2d, v0.2s, v1.s[0]3483; CHECK-NEXT: ret3484entry:3485 %shuffle = shufflevector <2 x i32> %v, <2 x i32> undef, <2 x i32> zeroinitializer3486 %vmull2.i = tail call <2 x i64> @llvm.aarch64.neon.smull.v2i64(<2 x i32> %a, <2 x i32> %shuffle)3487 ret <2 x i64> %vmull2.i3488}3489 3490define <4 x i32> @test_vmull_lane_u16_0(<4 x i16> %a, <4 x i16> %v) {3491; CHECK-LABEL: test_vmull_lane_u16_0:3492; CHECK: // %bb.0: // %entry3493; CHECK-NEXT: // kill: def $d1 killed $d1 def $q13494; CHECK-NEXT: umull v0.4s, v0.4h, v1.h[0]3495; CHECK-NEXT: ret3496entry:3497 %shuffle = shufflevector <4 x i16> %v, <4 x i16> undef, <4 x i32> zeroinitializer3498 %vmull2.i = tail call <4 x i32> @llvm.aarch64.neon.umull.v4i32(<4 x i16> %a, <4 x i16> %shuffle)3499 ret <4 x i32> %vmull2.i3500}3501 3502define <2 x i64> @test_vmull_lane_u32_0(<2 x i32> %a, <2 x i32> %v) {3503; CHECK-LABEL: test_vmull_lane_u32_0:3504; CHECK: // %bb.0: // %entry3505; CHECK-NEXT: // kill: def $d1 killed $d1 def $q13506; CHECK-NEXT: umull v0.2d, v0.2s, v1.s[0]3507; CHECK-NEXT: ret3508entry:3509 %shuffle = shufflevector <2 x i32> %v, <2 x i32> undef, <2 x i32> zeroinitializer3510 %vmull2.i = tail call <2 x i64> @llvm.aarch64.neon.umull.v2i64(<2 x i32> %a, <2 x i32> %shuffle)3511 ret <2 x i64> %vmull2.i3512}3513 3514define <4 x i32> @test_vmull_high_lane_s16_0(<8 x i16> %a, <4 x i16> %v) {3515; CHECK-SD-LABEL: test_vmull_high_lane_s16_0:3516; CHECK-SD: // %bb.0: // %entry3517; CHECK-SD-NEXT: // kill: def $d1 killed $d1 def $q13518; CHECK-SD-NEXT: smull2 v0.4s, v0.8h, v1.h[0]3519; CHECK-SD-NEXT: ret3520;3521; CHECK-GI-LABEL: test_vmull_high_lane_s16_0:3522; CHECK-GI: // %bb.0: // %entry3523; CHECK-GI-NEXT: mov d0, v0.d[1]3524; CHECK-GI-NEXT: // kill: def $d1 killed $d1 def $q13525; CHECK-GI-NEXT: smull v0.4s, v0.4h, v1.h[0]3526; CHECK-GI-NEXT: ret3527entry:3528 %shuffle.i = shufflevector <8 x i16> %a, <8 x i16> undef, <4 x i32> <i32 4, i32 5, i32 6, i32 7>3529 %shuffle = shufflevector <4 x i16> %v, <4 x i16> undef, <4 x i32> zeroinitializer3530 %vmull2.i = tail call <4 x i32> @llvm.aarch64.neon.smull.v4i32(<4 x i16> %shuffle.i, <4 x i16> %shuffle)3531 ret <4 x i32> %vmull2.i3532}3533 3534define <2 x i64> @test_vmull_high_lane_s32_0(<4 x i32> %a, <2 x i32> %v) {3535; CHECK-SD-LABEL: test_vmull_high_lane_s32_0:3536; CHECK-SD: // %bb.0: // %entry3537; CHECK-SD-NEXT: // kill: def $d1 killed $d1 def $q13538; CHECK-SD-NEXT: smull2 v0.2d, v0.4s, v1.s[0]3539; CHECK-SD-NEXT: ret3540;3541; CHECK-GI-LABEL: test_vmull_high_lane_s32_0:3542; CHECK-GI: // %bb.0: // %entry3543; CHECK-GI-NEXT: mov d0, v0.d[1]3544; CHECK-GI-NEXT: // kill: def $d1 killed $d1 def $q13545; CHECK-GI-NEXT: smull v0.2d, v0.2s, v1.s[0]3546; CHECK-GI-NEXT: ret3547entry:3548 %shuffle.i = shufflevector <4 x i32> %a, <4 x i32> undef, <2 x i32> <i32 2, i32 3>3549 %shuffle = shufflevector <2 x i32> %v, <2 x i32> undef, <2 x i32> zeroinitializer3550 %vmull2.i = tail call <2 x i64> @llvm.aarch64.neon.smull.v2i64(<2 x i32> %shuffle.i, <2 x i32> %shuffle)3551 ret <2 x i64> %vmull2.i3552}3553 3554define <4 x i32> @test_vmull_high_lane_u16_0(<8 x i16> %a, <4 x i16> %v) {3555; CHECK-SD-LABEL: test_vmull_high_lane_u16_0:3556; CHECK-SD: // %bb.0: // %entry3557; CHECK-SD-NEXT: // kill: def $d1 killed $d1 def $q13558; CHECK-SD-NEXT: umull2 v0.4s, v0.8h, v1.h[0]3559; CHECK-SD-NEXT: ret3560;3561; CHECK-GI-LABEL: test_vmull_high_lane_u16_0:3562; CHECK-GI: // %bb.0: // %entry3563; CHECK-GI-NEXT: mov d0, v0.d[1]3564; CHECK-GI-NEXT: // kill: def $d1 killed $d1 def $q13565; CHECK-GI-NEXT: umull v0.4s, v0.4h, v1.h[0]3566; CHECK-GI-NEXT: ret3567entry:3568 %shuffle.i = shufflevector <8 x i16> %a, <8 x i16> undef, <4 x i32> <i32 4, i32 5, i32 6, i32 7>3569 %shuffle = shufflevector <4 x i16> %v, <4 x i16> undef, <4 x i32> zeroinitializer3570 %vmull2.i = tail call <4 x i32> @llvm.aarch64.neon.umull.v4i32(<4 x i16> %shuffle.i, <4 x i16> %shuffle)3571 ret <4 x i32> %vmull2.i3572}3573 3574define <2 x i64> @test_vmull_high_lane_u32_0(<4 x i32> %a, <2 x i32> %v) {3575; CHECK-SD-LABEL: test_vmull_high_lane_u32_0:3576; CHECK-SD: // %bb.0: // %entry3577; CHECK-SD-NEXT: // kill: def $d1 killed $d1 def $q13578; CHECK-SD-NEXT: umull2 v0.2d, v0.4s, v1.s[0]3579; CHECK-SD-NEXT: ret3580;3581; CHECK-GI-LABEL: test_vmull_high_lane_u32_0:3582; CHECK-GI: // %bb.0: // %entry3583; CHECK-GI-NEXT: mov d0, v0.d[1]3584; CHECK-GI-NEXT: // kill: def $d1 killed $d1 def $q13585; CHECK-GI-NEXT: umull v0.2d, v0.2s, v1.s[0]3586; CHECK-GI-NEXT: ret3587entry:3588 %shuffle.i = shufflevector <4 x i32> %a, <4 x i32> undef, <2 x i32> <i32 2, i32 3>3589 %shuffle = shufflevector <2 x i32> %v, <2 x i32> undef, <2 x i32> zeroinitializer3590 %vmull2.i = tail call <2 x i64> @llvm.aarch64.neon.umull.v2i64(<2 x i32> %shuffle.i, <2 x i32> %shuffle)3591 ret <2 x i64> %vmull2.i3592}3593 3594define <4 x i32> @test_vmull_laneq_s16_0(<4 x i16> %a, <8 x i16> %v) {3595; CHECK-LABEL: test_vmull_laneq_s16_0:3596; CHECK: // %bb.0: // %entry3597; CHECK-NEXT: smull v0.4s, v0.4h, v1.h[0]3598; CHECK-NEXT: ret3599entry:3600 %shuffle = shufflevector <8 x i16> %v, <8 x i16> undef, <4 x i32> zeroinitializer3601 %vmull2.i = tail call <4 x i32> @llvm.aarch64.neon.smull.v4i32(<4 x i16> %a, <4 x i16> %shuffle)3602 ret <4 x i32> %vmull2.i3603}3604 3605define <2 x i64> @test_vmull_laneq_s32_0(<2 x i32> %a, <4 x i32> %v) {3606; CHECK-LABEL: test_vmull_laneq_s32_0:3607; CHECK: // %bb.0: // %entry3608; CHECK-NEXT: smull v0.2d, v0.2s, v1.s[0]3609; CHECK-NEXT: ret3610entry:3611 %shuffle = shufflevector <4 x i32> %v, <4 x i32> undef, <2 x i32> zeroinitializer3612 %vmull2.i = tail call <2 x i64> @llvm.aarch64.neon.smull.v2i64(<2 x i32> %a, <2 x i32> %shuffle)3613 ret <2 x i64> %vmull2.i3614}3615 3616define <4 x i32> @test_vmull_laneq_u16_0(<4 x i16> %a, <8 x i16> %v) {3617; CHECK-LABEL: test_vmull_laneq_u16_0:3618; CHECK: // %bb.0: // %entry3619; CHECK-NEXT: umull v0.4s, v0.4h, v1.h[0]3620; CHECK-NEXT: ret3621entry:3622 %shuffle = shufflevector <8 x i16> %v, <8 x i16> undef, <4 x i32> zeroinitializer3623 %vmull2.i = tail call <4 x i32> @llvm.aarch64.neon.umull.v4i32(<4 x i16> %a, <4 x i16> %shuffle)3624 ret <4 x i32> %vmull2.i3625}3626 3627define <2 x i64> @test_vmull_laneq_u32_0(<2 x i32> %a, <4 x i32> %v) {3628; CHECK-LABEL: test_vmull_laneq_u32_0:3629; CHECK: // %bb.0: // %entry3630; CHECK-NEXT: umull v0.2d, v0.2s, v1.s[0]3631; CHECK-NEXT: ret3632entry:3633 %shuffle = shufflevector <4 x i32> %v, <4 x i32> undef, <2 x i32> zeroinitializer3634 %vmull2.i = tail call <2 x i64> @llvm.aarch64.neon.umull.v2i64(<2 x i32> %a, <2 x i32> %shuffle)3635 ret <2 x i64> %vmull2.i3636}3637 3638define <4 x i32> @test_vmull_high_laneq_s16_0(<8 x i16> %a, <8 x i16> %v) {3639; CHECK-SD-LABEL: test_vmull_high_laneq_s16_0:3640; CHECK-SD: // %bb.0: // %entry3641; CHECK-SD-NEXT: smull2 v0.4s, v0.8h, v1.h[0]3642; CHECK-SD-NEXT: ret3643;3644; CHECK-GI-LABEL: test_vmull_high_laneq_s16_0:3645; CHECK-GI: // %bb.0: // %entry3646; CHECK-GI-NEXT: mov d0, v0.d[1]3647; CHECK-GI-NEXT: smull v0.4s, v0.4h, v1.h[0]3648; CHECK-GI-NEXT: ret3649entry:3650 %shuffle.i = shufflevector <8 x i16> %a, <8 x i16> undef, <4 x i32> <i32 4, i32 5, i32 6, i32 7>3651 %shuffle = shufflevector <8 x i16> %v, <8 x i16> undef, <4 x i32> zeroinitializer3652 %vmull2.i = tail call <4 x i32> @llvm.aarch64.neon.smull.v4i32(<4 x i16> %shuffle.i, <4 x i16> %shuffle)3653 ret <4 x i32> %vmull2.i3654}3655 3656define <2 x i64> @test_vmull_high_laneq_s32_0(<4 x i32> %a, <4 x i32> %v) {3657; CHECK-SD-LABEL: test_vmull_high_laneq_s32_0:3658; CHECK-SD: // %bb.0: // %entry3659; CHECK-SD-NEXT: smull2 v0.2d, v0.4s, v1.s[0]3660; CHECK-SD-NEXT: ret3661;3662; CHECK-GI-LABEL: test_vmull_high_laneq_s32_0:3663; CHECK-GI: // %bb.0: // %entry3664; CHECK-GI-NEXT: mov d0, v0.d[1]3665; CHECK-GI-NEXT: smull v0.2d, v0.2s, v1.s[0]3666; CHECK-GI-NEXT: ret3667entry:3668 %shuffle.i = shufflevector <4 x i32> %a, <4 x i32> undef, <2 x i32> <i32 2, i32 3>3669 %shuffle = shufflevector <4 x i32> %v, <4 x i32> undef, <2 x i32> zeroinitializer3670 %vmull2.i = tail call <2 x i64> @llvm.aarch64.neon.smull.v2i64(<2 x i32> %shuffle.i, <2 x i32> %shuffle)3671 ret <2 x i64> %vmull2.i3672}3673 3674define <4 x i32> @test_vmull_high_laneq_u16_0(<8 x i16> %a, <8 x i16> %v) {3675; CHECK-SD-LABEL: test_vmull_high_laneq_u16_0:3676; CHECK-SD: // %bb.0: // %entry3677; CHECK-SD-NEXT: umull2 v0.4s, v0.8h, v1.h[0]3678; CHECK-SD-NEXT: ret3679;3680; CHECK-GI-LABEL: test_vmull_high_laneq_u16_0:3681; CHECK-GI: // %bb.0: // %entry3682; CHECK-GI-NEXT: mov d0, v0.d[1]3683; CHECK-GI-NEXT: umull v0.4s, v0.4h, v1.h[0]3684; CHECK-GI-NEXT: ret3685entry:3686 %shuffle.i = shufflevector <8 x i16> %a, <8 x i16> undef, <4 x i32> <i32 4, i32 5, i32 6, i32 7>3687 %shuffle = shufflevector <8 x i16> %v, <8 x i16> undef, <4 x i32> zeroinitializer3688 %vmull2.i = tail call <4 x i32> @llvm.aarch64.neon.umull.v4i32(<4 x i16> %shuffle.i, <4 x i16> %shuffle)3689 ret <4 x i32> %vmull2.i3690}3691 3692define <2 x i64> @test_vmull_high_laneq_u32_0(<4 x i32> %a, <4 x i32> %v) {3693; CHECK-SD-LABEL: test_vmull_high_laneq_u32_0:3694; CHECK-SD: // %bb.0: // %entry3695; CHECK-SD-NEXT: umull2 v0.2d, v0.4s, v1.s[0]3696; CHECK-SD-NEXT: ret3697;3698; CHECK-GI-LABEL: test_vmull_high_laneq_u32_0:3699; CHECK-GI: // %bb.0: // %entry3700; CHECK-GI-NEXT: mov d0, v0.d[1]3701; CHECK-GI-NEXT: umull v0.2d, v0.2s, v1.s[0]3702; CHECK-GI-NEXT: ret3703entry:3704 %shuffle.i = shufflevector <4 x i32> %a, <4 x i32> undef, <2 x i32> <i32 2, i32 3>3705 %shuffle = shufflevector <4 x i32> %v, <4 x i32> undef, <2 x i32> zeroinitializer3706 %vmull2.i = tail call <2 x i64> @llvm.aarch64.neon.umull.v2i64(<2 x i32> %shuffle.i, <2 x i32> %shuffle)3707 ret <2 x i64> %vmull2.i3708}3709 3710define <4 x i32> @test_vqdmlal_lane_s16_0(<4 x i32> %a, <4 x i16> %b, <4 x i16> %v) {3711; CHECK-LABEL: test_vqdmlal_lane_s16_0:3712; CHECK: // %bb.0: // %entry3713; CHECK-NEXT: // kill: def $d2 killed $d2 def $q23714; CHECK-NEXT: sqdmlal v0.4s, v1.4h, v2.h[0]3715; CHECK-NEXT: ret3716entry:3717 %shuffle = shufflevector <4 x i16> %v, <4 x i16> undef, <4 x i32> zeroinitializer3718 %vqdmlal2.i = tail call <4 x i32> @llvm.aarch64.neon.sqdmull.v4i32(<4 x i16> %b, <4 x i16> %shuffle)3719 %vqdmlal4.i = tail call <4 x i32> @llvm.aarch64.neon.sqadd.v4i32(<4 x i32> %a, <4 x i32> %vqdmlal2.i)3720 ret <4 x i32> %vqdmlal4.i3721}3722 3723define <2 x i64> @test_vqdmlal_lane_s32_0(<2 x i64> %a, <2 x i32> %b, <2 x i32> %v) {3724; CHECK-LABEL: test_vqdmlal_lane_s32_0:3725; CHECK: // %bb.0: // %entry3726; CHECK-NEXT: // kill: def $d2 killed $d2 def $q23727; CHECK-NEXT: sqdmlal v0.2d, v1.2s, v2.s[0]3728; CHECK-NEXT: ret3729entry:3730 %shuffle = shufflevector <2 x i32> %v, <2 x i32> undef, <2 x i32> zeroinitializer3731 %vqdmlal2.i = tail call <2 x i64> @llvm.aarch64.neon.sqdmull.v2i64(<2 x i32> %b, <2 x i32> %shuffle)3732 %vqdmlal4.i = tail call <2 x i64> @llvm.aarch64.neon.sqadd.v2i64(<2 x i64> %a, <2 x i64> %vqdmlal2.i)3733 ret <2 x i64> %vqdmlal4.i3734}3735 3736define <4 x i32> @test_vqdmlal_high_lane_s16_0(<4 x i32> %a, <8 x i16> %b, <4 x i16> %v) {3737; CHECK-SD-LABEL: test_vqdmlal_high_lane_s16_0:3738; CHECK-SD: // %bb.0: // %entry3739; CHECK-SD-NEXT: // kill: def $d2 killed $d2 def $q23740; CHECK-SD-NEXT: sqdmlal2 v0.4s, v1.8h, v2.h[0]3741; CHECK-SD-NEXT: ret3742;3743; CHECK-GI-LABEL: test_vqdmlal_high_lane_s16_0:3744; CHECK-GI: // %bb.0: // %entry3745; CHECK-GI-NEXT: mov d1, v1.d[1]3746; CHECK-GI-NEXT: // kill: def $d2 killed $d2 def $q23747; CHECK-GI-NEXT: sqdmlal v0.4s, v1.4h, v2.h[0]3748; CHECK-GI-NEXT: ret3749entry:3750 %shuffle.i = shufflevector <8 x i16> %b, <8 x i16> undef, <4 x i32> <i32 4, i32 5, i32 6, i32 7>3751 %shuffle = shufflevector <4 x i16> %v, <4 x i16> undef, <4 x i32> zeroinitializer3752 %vqdmlal2.i = tail call <4 x i32> @llvm.aarch64.neon.sqdmull.v4i32(<4 x i16> %shuffle.i, <4 x i16> %shuffle)3753 %vqdmlal4.i = tail call <4 x i32> @llvm.aarch64.neon.sqadd.v4i32(<4 x i32> %a, <4 x i32> %vqdmlal2.i)3754 ret <4 x i32> %vqdmlal4.i3755}3756 3757define <2 x i64> @test_vqdmlal_high_lane_s32_0(<2 x i64> %a, <4 x i32> %b, <2 x i32> %v) {3758; CHECK-SD-LABEL: test_vqdmlal_high_lane_s32_0:3759; CHECK-SD: // %bb.0: // %entry3760; CHECK-SD-NEXT: // kill: def $d2 killed $d2 def $q23761; CHECK-SD-NEXT: sqdmlal2 v0.2d, v1.4s, v2.s[0]3762; CHECK-SD-NEXT: ret3763;3764; CHECK-GI-LABEL: test_vqdmlal_high_lane_s32_0:3765; CHECK-GI: // %bb.0: // %entry3766; CHECK-GI-NEXT: mov d1, v1.d[1]3767; CHECK-GI-NEXT: // kill: def $d2 killed $d2 def $q23768; CHECK-GI-NEXT: sqdmlal v0.2d, v1.2s, v2.s[0]3769; CHECK-GI-NEXT: ret3770entry:3771 %shuffle.i = shufflevector <4 x i32> %b, <4 x i32> undef, <2 x i32> <i32 2, i32 3>3772 %shuffle = shufflevector <2 x i32> %v, <2 x i32> undef, <2 x i32> zeroinitializer3773 %vqdmlal2.i = tail call <2 x i64> @llvm.aarch64.neon.sqdmull.v2i64(<2 x i32> %shuffle.i, <2 x i32> %shuffle)3774 %vqdmlal4.i = tail call <2 x i64> @llvm.aarch64.neon.sqadd.v2i64(<2 x i64> %a, <2 x i64> %vqdmlal2.i)3775 ret <2 x i64> %vqdmlal4.i3776}3777 3778define <4 x i32> @test_vqdmlsl_lane_s16_0(<4 x i32> %a, <4 x i16> %b, <4 x i16> %v) {3779; CHECK-LABEL: test_vqdmlsl_lane_s16_0:3780; CHECK: // %bb.0: // %entry3781; CHECK-NEXT: // kill: def $d2 killed $d2 def $q23782; CHECK-NEXT: sqdmlsl v0.4s, v1.4h, v2.h[0]3783; CHECK-NEXT: ret3784entry:3785 %shuffle = shufflevector <4 x i16> %v, <4 x i16> undef, <4 x i32> zeroinitializer3786 %vqdmlsl2.i = tail call <4 x i32> @llvm.aarch64.neon.sqdmull.v4i32(<4 x i16> %b, <4 x i16> %shuffle)3787 %vqdmlsl4.i = tail call <4 x i32> @llvm.aarch64.neon.sqsub.v4i32(<4 x i32> %a, <4 x i32> %vqdmlsl2.i)3788 ret <4 x i32> %vqdmlsl4.i3789}3790 3791define <2 x i64> @test_vqdmlsl_lane_s32_0(<2 x i64> %a, <2 x i32> %b, <2 x i32> %v) {3792; CHECK-LABEL: test_vqdmlsl_lane_s32_0:3793; CHECK: // %bb.0: // %entry3794; CHECK-NEXT: // kill: def $d2 killed $d2 def $q23795; CHECK-NEXT: sqdmlsl v0.2d, v1.2s, v2.s[0]3796; CHECK-NEXT: ret3797entry:3798 %shuffle = shufflevector <2 x i32> %v, <2 x i32> undef, <2 x i32> zeroinitializer3799 %vqdmlsl2.i = tail call <2 x i64> @llvm.aarch64.neon.sqdmull.v2i64(<2 x i32> %b, <2 x i32> %shuffle)3800 %vqdmlsl4.i = tail call <2 x i64> @llvm.aarch64.neon.sqsub.v2i64(<2 x i64> %a, <2 x i64> %vqdmlsl2.i)3801 ret <2 x i64> %vqdmlsl4.i3802}3803 3804define <4 x i32> @test_vqdmlsl_high_lane_s16_0(<4 x i32> %a, <8 x i16> %b, <4 x i16> %v) {3805; CHECK-SD-LABEL: test_vqdmlsl_high_lane_s16_0:3806; CHECK-SD: // %bb.0: // %entry3807; CHECK-SD-NEXT: // kill: def $d2 killed $d2 def $q23808; CHECK-SD-NEXT: sqdmlsl2 v0.4s, v1.8h, v2.h[0]3809; CHECK-SD-NEXT: ret3810;3811; CHECK-GI-LABEL: test_vqdmlsl_high_lane_s16_0:3812; CHECK-GI: // %bb.0: // %entry3813; CHECK-GI-NEXT: mov d1, v1.d[1]3814; CHECK-GI-NEXT: // kill: def $d2 killed $d2 def $q23815; CHECK-GI-NEXT: sqdmlsl v0.4s, v1.4h, v2.h[0]3816; CHECK-GI-NEXT: ret3817entry:3818 %shuffle.i = shufflevector <8 x i16> %b, <8 x i16> undef, <4 x i32> <i32 4, i32 5, i32 6, i32 7>3819 %shuffle = shufflevector <4 x i16> %v, <4 x i16> undef, <4 x i32> zeroinitializer3820 %vqdmlsl2.i = tail call <4 x i32> @llvm.aarch64.neon.sqdmull.v4i32(<4 x i16> %shuffle.i, <4 x i16> %shuffle)3821 %vqdmlsl4.i = tail call <4 x i32> @llvm.aarch64.neon.sqsub.v4i32(<4 x i32> %a, <4 x i32> %vqdmlsl2.i)3822 ret <4 x i32> %vqdmlsl4.i3823}3824 3825define <2 x i64> @test_vqdmlsl_high_lane_s32_0(<2 x i64> %a, <4 x i32> %b, <2 x i32> %v) {3826; CHECK-SD-LABEL: test_vqdmlsl_high_lane_s32_0:3827; CHECK-SD: // %bb.0: // %entry3828; CHECK-SD-NEXT: // kill: def $d2 killed $d2 def $q23829; CHECK-SD-NEXT: sqdmlsl2 v0.2d, v1.4s, v2.s[0]3830; CHECK-SD-NEXT: ret3831;3832; CHECK-GI-LABEL: test_vqdmlsl_high_lane_s32_0:3833; CHECK-GI: // %bb.0: // %entry3834; CHECK-GI-NEXT: mov d1, v1.d[1]3835; CHECK-GI-NEXT: // kill: def $d2 killed $d2 def $q23836; CHECK-GI-NEXT: sqdmlsl v0.2d, v1.2s, v2.s[0]3837; CHECK-GI-NEXT: ret3838entry:3839 %shuffle.i = shufflevector <4 x i32> %b, <4 x i32> undef, <2 x i32> <i32 2, i32 3>3840 %shuffle = shufflevector <2 x i32> %v, <2 x i32> undef, <2 x i32> zeroinitializer3841 %vqdmlsl2.i = tail call <2 x i64> @llvm.aarch64.neon.sqdmull.v2i64(<2 x i32> %shuffle.i, <2 x i32> %shuffle)3842 %vqdmlsl4.i = tail call <2 x i64> @llvm.aarch64.neon.sqsub.v2i64(<2 x i64> %a, <2 x i64> %vqdmlsl2.i)3843 ret <2 x i64> %vqdmlsl4.i3844}3845 3846define <4 x i32> @test_vqdmull_lane_s16_0(<4 x i16> %a, <4 x i16> %v) {3847; CHECK-LABEL: test_vqdmull_lane_s16_0:3848; CHECK: // %bb.0: // %entry3849; CHECK-NEXT: // kill: def $d1 killed $d1 def $q13850; CHECK-NEXT: sqdmull v0.4s, v0.4h, v1.h[0]3851; CHECK-NEXT: ret3852entry:3853 %shuffle = shufflevector <4 x i16> %v, <4 x i16> undef, <4 x i32> zeroinitializer3854 %vqdmull2.i = tail call <4 x i32> @llvm.aarch64.neon.sqdmull.v4i32(<4 x i16> %a, <4 x i16> %shuffle)3855 ret <4 x i32> %vqdmull2.i3856}3857 3858define <2 x i64> @test_vqdmull_lane_s32_0(<2 x i32> %a, <2 x i32> %v) {3859; CHECK-LABEL: test_vqdmull_lane_s32_0:3860; CHECK: // %bb.0: // %entry3861; CHECK-NEXT: // kill: def $d1 killed $d1 def $q13862; CHECK-NEXT: sqdmull v0.2d, v0.2s, v1.s[0]3863; CHECK-NEXT: ret3864entry:3865 %shuffle = shufflevector <2 x i32> %v, <2 x i32> undef, <2 x i32> zeroinitializer3866 %vqdmull2.i = tail call <2 x i64> @llvm.aarch64.neon.sqdmull.v2i64(<2 x i32> %a, <2 x i32> %shuffle)3867 ret <2 x i64> %vqdmull2.i3868}3869 3870define <4 x i32> @test_vqdmull_laneq_s16_0(<4 x i16> %a, <8 x i16> %v) {3871; CHECK-LABEL: test_vqdmull_laneq_s16_0:3872; CHECK: // %bb.0: // %entry3873; CHECK-NEXT: sqdmull v0.4s, v0.4h, v1.h[0]3874; CHECK-NEXT: ret3875entry:3876 %shuffle = shufflevector <8 x i16> %v, <8 x i16> undef, <4 x i32> zeroinitializer3877 %vqdmull2.i = tail call <4 x i32> @llvm.aarch64.neon.sqdmull.v4i32(<4 x i16> %a, <4 x i16> %shuffle)3878 ret <4 x i32> %vqdmull2.i3879}3880 3881define <2 x i64> @test_vqdmull_laneq_s32_0(<2 x i32> %a, <4 x i32> %v) {3882; CHECK-LABEL: test_vqdmull_laneq_s32_0:3883; CHECK: // %bb.0: // %entry3884; CHECK-NEXT: sqdmull v0.2d, v0.2s, v1.s[0]3885; CHECK-NEXT: ret3886entry:3887 %shuffle = shufflevector <4 x i32> %v, <4 x i32> undef, <2 x i32> zeroinitializer3888 %vqdmull2.i = tail call <2 x i64> @llvm.aarch64.neon.sqdmull.v2i64(<2 x i32> %a, <2 x i32> %shuffle)3889 ret <2 x i64> %vqdmull2.i3890}3891 3892define <4 x i32> @test_vqdmull_high_lane_s16_0(<8 x i16> %a, <4 x i16> %v) {3893; CHECK-SD-LABEL: test_vqdmull_high_lane_s16_0:3894; CHECK-SD: // %bb.0: // %entry3895; CHECK-SD-NEXT: // kill: def $d1 killed $d1 def $q13896; CHECK-SD-NEXT: sqdmull2 v0.4s, v0.8h, v1.h[0]3897; CHECK-SD-NEXT: ret3898;3899; CHECK-GI-LABEL: test_vqdmull_high_lane_s16_0:3900; CHECK-GI: // %bb.0: // %entry3901; CHECK-GI-NEXT: mov d0, v0.d[1]3902; CHECK-GI-NEXT: // kill: def $d1 killed $d1 def $q13903; CHECK-GI-NEXT: sqdmull v0.4s, v0.4h, v1.h[0]3904; CHECK-GI-NEXT: ret3905entry:3906 %shuffle.i = shufflevector <8 x i16> %a, <8 x i16> undef, <4 x i32> <i32 4, i32 5, i32 6, i32 7>3907 %shuffle = shufflevector <4 x i16> %v, <4 x i16> undef, <4 x i32> zeroinitializer3908 %vqdmull2.i = tail call <4 x i32> @llvm.aarch64.neon.sqdmull.v4i32(<4 x i16> %shuffle.i, <4 x i16> %shuffle)3909 ret <4 x i32> %vqdmull2.i3910}3911 3912define <2 x i64> @test_vqdmull_high_lane_s32_0(<4 x i32> %a, <2 x i32> %v) {3913; CHECK-SD-LABEL: test_vqdmull_high_lane_s32_0:3914; CHECK-SD: // %bb.0: // %entry3915; CHECK-SD-NEXT: // kill: def $d1 killed $d1 def $q13916; CHECK-SD-NEXT: sqdmull2 v0.2d, v0.4s, v1.s[0]3917; CHECK-SD-NEXT: ret3918;3919; CHECK-GI-LABEL: test_vqdmull_high_lane_s32_0:3920; CHECK-GI: // %bb.0: // %entry3921; CHECK-GI-NEXT: mov d0, v0.d[1]3922; CHECK-GI-NEXT: // kill: def $d1 killed $d1 def $q13923; CHECK-GI-NEXT: sqdmull v0.2d, v0.2s, v1.s[0]3924; CHECK-GI-NEXT: ret3925entry:3926 %shuffle.i = shufflevector <4 x i32> %a, <4 x i32> undef, <2 x i32> <i32 2, i32 3>3927 %shuffle = shufflevector <2 x i32> %v, <2 x i32> undef, <2 x i32> zeroinitializer3928 %vqdmull2.i = tail call <2 x i64> @llvm.aarch64.neon.sqdmull.v2i64(<2 x i32> %shuffle.i, <2 x i32> %shuffle)3929 ret <2 x i64> %vqdmull2.i3930}3931 3932define <4 x i32> @test_vqdmull_high_laneq_s16_0(<8 x i16> %a, <8 x i16> %v) {3933; CHECK-SD-LABEL: test_vqdmull_high_laneq_s16_0:3934; CHECK-SD: // %bb.0: // %entry3935; CHECK-SD-NEXT: sqdmull2 v0.4s, v0.8h, v1.h[0]3936; CHECK-SD-NEXT: ret3937;3938; CHECK-GI-LABEL: test_vqdmull_high_laneq_s16_0:3939; CHECK-GI: // %bb.0: // %entry3940; CHECK-GI-NEXT: mov d0, v0.d[1]3941; CHECK-GI-NEXT: sqdmull v0.4s, v0.4h, v1.h[0]3942; CHECK-GI-NEXT: ret3943entry:3944 %shuffle.i = shufflevector <8 x i16> %a, <8 x i16> undef, <4 x i32> <i32 4, i32 5, i32 6, i32 7>3945 %shuffle = shufflevector <8 x i16> %v, <8 x i16> undef, <4 x i32> zeroinitializer3946 %vqdmull2.i = tail call <4 x i32> @llvm.aarch64.neon.sqdmull.v4i32(<4 x i16> %shuffle.i, <4 x i16> %shuffle)3947 ret <4 x i32> %vqdmull2.i3948}3949 3950define <2 x i64> @test_vqdmull_high_laneq_s32_0(<4 x i32> %a, <4 x i32> %v) {3951; CHECK-SD-LABEL: test_vqdmull_high_laneq_s32_0:3952; CHECK-SD: // %bb.0: // %entry3953; CHECK-SD-NEXT: sqdmull2 v0.2d, v0.4s, v1.s[0]3954; CHECK-SD-NEXT: ret3955;3956; CHECK-GI-LABEL: test_vqdmull_high_laneq_s32_0:3957; CHECK-GI: // %bb.0: // %entry3958; CHECK-GI-NEXT: mov d0, v0.d[1]3959; CHECK-GI-NEXT: sqdmull v0.2d, v0.2s, v1.s[0]3960; CHECK-GI-NEXT: ret3961entry:3962 %shuffle.i = shufflevector <4 x i32> %a, <4 x i32> undef, <2 x i32> <i32 2, i32 3>3963 %shuffle = shufflevector <4 x i32> %v, <4 x i32> undef, <2 x i32> zeroinitializer3964 %vqdmull2.i = tail call <2 x i64> @llvm.aarch64.neon.sqdmull.v2i64(<2 x i32> %shuffle.i, <2 x i32> %shuffle)3965 ret <2 x i64> %vqdmull2.i3966}3967 3968define <4 x i16> @test_vqdmulh_lane_s16_0(<4 x i16> %a, <4 x i16> %v) {3969; CHECK-LABEL: test_vqdmulh_lane_s16_0:3970; CHECK: // %bb.0: // %entry3971; CHECK-NEXT: // kill: def $d1 killed $d1 def $q13972; CHECK-NEXT: sqdmulh v0.4h, v0.4h, v1.h[0]3973; CHECK-NEXT: ret3974entry:3975 %shuffle = shufflevector <4 x i16> %v, <4 x i16> undef, <4 x i32> zeroinitializer3976 %vqdmulh2.i = tail call <4 x i16> @llvm.aarch64.neon.sqdmulh.v4i16(<4 x i16> %a, <4 x i16> %shuffle)3977 ret <4 x i16> %vqdmulh2.i3978}3979 3980define <8 x i16> @test_vqdmulhq_lane_s16_0(<8 x i16> %a, <4 x i16> %v) {3981; CHECK-LABEL: test_vqdmulhq_lane_s16_0:3982; CHECK: // %bb.0: // %entry3983; CHECK-NEXT: // kill: def $d1 killed $d1 def $q13984; CHECK-NEXT: sqdmulh v0.8h, v0.8h, v1.h[0]3985; CHECK-NEXT: ret3986entry:3987 %shuffle = shufflevector <4 x i16> %v, <4 x i16> undef, <8 x i32> zeroinitializer3988 %vqdmulh2.i = tail call <8 x i16> @llvm.aarch64.neon.sqdmulh.v8i16(<8 x i16> %a, <8 x i16> %shuffle)3989 ret <8 x i16> %vqdmulh2.i3990}3991 3992define <2 x i32> @test_vqdmulh_lane_s32_0(<2 x i32> %a, <2 x i32> %v) {3993; CHECK-LABEL: test_vqdmulh_lane_s32_0:3994; CHECK: // %bb.0: // %entry3995; CHECK-NEXT: // kill: def $d1 killed $d1 def $q13996; CHECK-NEXT: sqdmulh v0.2s, v0.2s, v1.s[0]3997; CHECK-NEXT: ret3998entry:3999 %shuffle = shufflevector <2 x i32> %v, <2 x i32> undef, <2 x i32> zeroinitializer4000 %vqdmulh2.i = tail call <2 x i32> @llvm.aarch64.neon.sqdmulh.v2i32(<2 x i32> %a, <2 x i32> %shuffle)4001 ret <2 x i32> %vqdmulh2.i4002}4003 4004define <4 x i32> @test_vqdmulhq_lane_s32_0(<4 x i32> %a, <2 x i32> %v) {4005; CHECK-LABEL: test_vqdmulhq_lane_s32_0:4006; CHECK: // %bb.0: // %entry4007; CHECK-NEXT: // kill: def $d1 killed $d1 def $q14008; CHECK-NEXT: sqdmulh v0.4s, v0.4s, v1.s[0]4009; CHECK-NEXT: ret4010entry:4011 %shuffle = shufflevector <2 x i32> %v, <2 x i32> undef, <4 x i32> zeroinitializer4012 %vqdmulh2.i = tail call <4 x i32> @llvm.aarch64.neon.sqdmulh.v4i32(<4 x i32> %a, <4 x i32> %shuffle)4013 ret <4 x i32> %vqdmulh2.i4014}4015 4016define <4 x i16> @test_vqrdmulh_lane_s16_0(<4 x i16> %a, <4 x i16> %v) {4017; CHECK-LABEL: test_vqrdmulh_lane_s16_0:4018; CHECK: // %bb.0: // %entry4019; CHECK-NEXT: // kill: def $d1 killed $d1 def $q14020; CHECK-NEXT: sqrdmulh v0.4h, v0.4h, v1.h[0]4021; CHECK-NEXT: ret4022entry:4023 %shuffle = shufflevector <4 x i16> %v, <4 x i16> undef, <4 x i32> zeroinitializer4024 %vqrdmulh2.i = tail call <4 x i16> @llvm.aarch64.neon.sqrdmulh.v4i16(<4 x i16> %a, <4 x i16> %shuffle)4025 ret <4 x i16> %vqrdmulh2.i4026}4027 4028define <8 x i16> @test_vqrdmulhq_lane_s16_0(<8 x i16> %a, <4 x i16> %v) {4029; CHECK-LABEL: test_vqrdmulhq_lane_s16_0:4030; CHECK: // %bb.0: // %entry4031; CHECK-NEXT: // kill: def $d1 killed $d1 def $q14032; CHECK-NEXT: sqrdmulh v0.8h, v0.8h, v1.h[0]4033; CHECK-NEXT: ret4034entry:4035 %shuffle = shufflevector <4 x i16> %v, <4 x i16> undef, <8 x i32> zeroinitializer4036 %vqrdmulh2.i = tail call <8 x i16> @llvm.aarch64.neon.sqrdmulh.v8i16(<8 x i16> %a, <8 x i16> %shuffle)4037 ret <8 x i16> %vqrdmulh2.i4038}4039 4040define <2 x i32> @test_vqrdmulh_lane_s32_0(<2 x i32> %a, <2 x i32> %v) {4041; CHECK-LABEL: test_vqrdmulh_lane_s32_0:4042; CHECK: // %bb.0: // %entry4043; CHECK-NEXT: // kill: def $d1 killed $d1 def $q14044; CHECK-NEXT: sqrdmulh v0.2s, v0.2s, v1.s[0]4045; CHECK-NEXT: ret4046entry:4047 %shuffle = shufflevector <2 x i32> %v, <2 x i32> undef, <2 x i32> zeroinitializer4048 %vqrdmulh2.i = tail call <2 x i32> @llvm.aarch64.neon.sqrdmulh.v2i32(<2 x i32> %a, <2 x i32> %shuffle)4049 ret <2 x i32> %vqrdmulh2.i4050}4051 4052define <4 x i32> @test_vqrdmulhq_lane_s32_0(<4 x i32> %a, <2 x i32> %v) {4053; CHECK-LABEL: test_vqrdmulhq_lane_s32_0:4054; CHECK: // %bb.0: // %entry4055; CHECK-NEXT: // kill: def $d1 killed $d1 def $q14056; CHECK-NEXT: sqrdmulh v0.4s, v0.4s, v1.s[0]4057; CHECK-NEXT: ret4058entry:4059 %shuffle = shufflevector <2 x i32> %v, <2 x i32> undef, <4 x i32> zeroinitializer4060 %vqrdmulh2.i = tail call <4 x i32> @llvm.aarch64.neon.sqrdmulh.v4i32(<4 x i32> %a, <4 x i32> %shuffle)4061 ret <4 x i32> %vqrdmulh2.i4062}4063 4064define <2 x float> @test_vmul_lane_f32_0(<2 x float> %a, <2 x float> %v) {4065; CHECK-LABEL: test_vmul_lane_f32_0:4066; CHECK: // %bb.0: // %entry4067; CHECK-NEXT: // kill: def $d1 killed $d1 def $q14068; CHECK-NEXT: fmul v0.2s, v0.2s, v1.s[0]4069; CHECK-NEXT: ret4070entry:4071 %shuffle = shufflevector <2 x float> %v, <2 x float> undef, <2 x i32> zeroinitializer4072 %mul = fmul <2 x float> %shuffle, %a4073 ret <2 x float> %mul4074}4075 4076define <4 x float> @test_vmulq_lane_f32_0(<4 x float> %a, <2 x float> %v) {4077; CHECK-LABEL: test_vmulq_lane_f32_0:4078; CHECK: // %bb.0: // %entry4079; CHECK-NEXT: // kill: def $d1 killed $d1 def $q14080; CHECK-NEXT: fmul v0.4s, v0.4s, v1.s[0]4081; CHECK-NEXT: ret4082entry:4083 %shuffle = shufflevector <2 x float> %v, <2 x float> undef, <4 x i32> zeroinitializer4084 %mul = fmul <4 x float> %shuffle, %a4085 ret <4 x float> %mul4086}4087 4088define <2 x float> @test_vmul_laneq_f32_0(<2 x float> %a, <4 x float> %v) {4089; CHECK-LABEL: test_vmul_laneq_f32_0:4090; CHECK: // %bb.0: // %entry4091; CHECK-NEXT: fmul v0.2s, v0.2s, v1.s[0]4092; CHECK-NEXT: ret4093entry:4094 %shuffle = shufflevector <4 x float> %v, <4 x float> undef, <2 x i32> zeroinitializer4095 %mul = fmul <2 x float> %shuffle, %a4096 ret <2 x float> %mul4097}4098 4099define <1 x double> @test_vmul_laneq_f64_0(<1 x double> %a, <2 x double> %v) {4100; CHECK-LABEL: test_vmul_laneq_f64_0:4101; CHECK: // %bb.0: // %entry4102; CHECK-NEXT: fmul d0, d0, d14103; CHECK-NEXT: ret4104entry:4105 %0 = bitcast <1 x double> %a to <8 x i8>4106 %1 = bitcast <8 x i8> %0 to double4107 %extract = extractelement <2 x double> %v, i32 04108 %2 = fmul double %1, %extract4109 %3 = insertelement <1 x double> undef, double %2, i32 04110 ret <1 x double> %34111}4112 4113define <4 x float> @test_vmulq_laneq_f32_0(<4 x float> %a, <4 x float> %v) {4114; CHECK-LABEL: test_vmulq_laneq_f32_0:4115; CHECK: // %bb.0: // %entry4116; CHECK-NEXT: fmul v0.4s, v0.4s, v1.s[0]4117; CHECK-NEXT: ret4118entry:4119 %shuffle = shufflevector <4 x float> %v, <4 x float> undef, <4 x i32> zeroinitializer4120 %mul = fmul <4 x float> %shuffle, %a4121 ret <4 x float> %mul4122}4123 4124define <2 x double> @test_vmulq_laneq_f64_0(<2 x double> %a, <2 x double> %v) {4125; CHECK-LABEL: test_vmulq_laneq_f64_0:4126; CHECK: // %bb.0: // %entry4127; CHECK-NEXT: fmul v0.2d, v0.2d, v1.d[0]4128; CHECK-NEXT: ret4129entry:4130 %shuffle = shufflevector <2 x double> %v, <2 x double> undef, <2 x i32> zeroinitializer4131 %mul = fmul <2 x double> %shuffle, %a4132 ret <2 x double> %mul4133}4134 4135define <2 x float> @test_vmulx_lane_f32_0(<2 x float> %a, <2 x float> %v) {4136; CHECK-LABEL: test_vmulx_lane_f32_0:4137; CHECK: // %bb.0: // %entry4138; CHECK-NEXT: // kill: def $d1 killed $d1 def $q14139; CHECK-NEXT: fmulx v0.2s, v0.2s, v1.s[0]4140; CHECK-NEXT: ret4141entry:4142 %shuffle = shufflevector <2 x float> %v, <2 x float> undef, <2 x i32> zeroinitializer4143 %vmulx2.i = tail call <2 x float> @llvm.aarch64.neon.fmulx.v2f32(<2 x float> %a, <2 x float> %shuffle)4144 ret <2 x float> %vmulx2.i4145}4146 4147define <4 x float> @test_vmulxq_lane_f32_0(<4 x float> %a, <2 x float> %v) {4148; CHECK-LABEL: test_vmulxq_lane_f32_0:4149; CHECK: // %bb.0: // %entry4150; CHECK-NEXT: // kill: def $d1 killed $d1 def $q14151; CHECK-NEXT: fmulx v0.4s, v0.4s, v1.s[0]4152; CHECK-NEXT: ret4153entry:4154 %shuffle = shufflevector <2 x float> %v, <2 x float> undef, <4 x i32> zeroinitializer4155 %vmulx2.i = tail call <4 x float> @llvm.aarch64.neon.fmulx.v4f32(<4 x float> %a, <4 x float> %shuffle)4156 ret <4 x float> %vmulx2.i4157}4158 4159define <2 x double> @test_vmulxq_lane_f64_0(<2 x double> %a, <1 x double> %v) {4160; CHECK-SD-LABEL: test_vmulxq_lane_f64_0:4161; CHECK-SD: // %bb.0: // %entry4162; CHECK-SD-NEXT: // kill: def $d1 killed $d1 def $q14163; CHECK-SD-NEXT: fmulx v0.2d, v0.2d, v1.d[0]4164; CHECK-SD-NEXT: ret4165;4166; CHECK-GI-LABEL: test_vmulxq_lane_f64_0:4167; CHECK-GI: // %bb.0: // %entry4168; CHECK-GI-NEXT: // kill: def $d1 killed $d1 def $q14169; CHECK-GI-NEXT: dup v1.2d, v1.d[0]4170; CHECK-GI-NEXT: fmulx v0.2d, v0.2d, v1.2d4171; CHECK-GI-NEXT: ret4172entry:4173 %shuffle = shufflevector <1 x double> %v, <1 x double> undef, <2 x i32> zeroinitializer4174 %vmulx2.i = tail call <2 x double> @llvm.aarch64.neon.fmulx.v2f64(<2 x double> %a, <2 x double> %shuffle)4175 ret <2 x double> %vmulx2.i4176}4177 4178define <2 x float> @test_vmulx_laneq_f32_0(<2 x float> %a, <4 x float> %v) {4179; CHECK-LABEL: test_vmulx_laneq_f32_0:4180; CHECK: // %bb.0: // %entry4181; CHECK-NEXT: fmulx v0.2s, v0.2s, v1.s[0]4182; CHECK-NEXT: ret4183entry:4184 %shuffle = shufflevector <4 x float> %v, <4 x float> undef, <2 x i32> zeroinitializer4185 %vmulx2.i = tail call <2 x float> @llvm.aarch64.neon.fmulx.v2f32(<2 x float> %a, <2 x float> %shuffle)4186 ret <2 x float> %vmulx2.i4187}4188 4189define <4 x float> @test_vmulxq_laneq_f32_0(<4 x float> %a, <4 x float> %v) {4190; CHECK-LABEL: test_vmulxq_laneq_f32_0:4191; CHECK: // %bb.0: // %entry4192; CHECK-NEXT: fmulx v0.4s, v0.4s, v1.s[0]4193; CHECK-NEXT: ret4194entry:4195 %shuffle = shufflevector <4 x float> %v, <4 x float> undef, <4 x i32> zeroinitializer4196 %vmulx2.i = tail call <4 x float> @llvm.aarch64.neon.fmulx.v4f32(<4 x float> %a, <4 x float> %shuffle)4197 ret <4 x float> %vmulx2.i4198}4199 4200define <1 x double> @test_vmulx_laneq_f64_0(<1 x double> %a, <2 x double> %v) {4201; CHECK-LABEL: test_vmulx_laneq_f64_0:4202; CHECK: // %bb.0: // %entry4203; CHECK-NEXT: fmulx d0, d0, d14204; CHECK-NEXT: ret4205entry:4206 %vget_lane = extractelement <1 x double> %a, i64 04207 %vgetq_lane = extractelement <2 x double> %v, i64 04208 %vmulxd_f64.i = tail call double @llvm.aarch64.neon.fmulx.f64(double %vget_lane, double %vgetq_lane)4209 %vset_lane = insertelement <1 x double> poison, double %vmulxd_f64.i, i64 04210 ret <1 x double> %vset_lane4211}4212 4213define <2 x double> @test_vmulxq_laneq_f64_0(<2 x double> %a, <2 x double> %v) {4214; CHECK-LABEL: test_vmulxq_laneq_f64_0:4215; CHECK: // %bb.0: // %entry4216; CHECK-NEXT: fmulx v0.2d, v0.2d, v1.d[0]4217; CHECK-NEXT: ret4218entry:4219 %shuffle = shufflevector <2 x double> %v, <2 x double> undef, <2 x i32> zeroinitializer4220 %vmulx2.i = tail call <2 x double> @llvm.aarch64.neon.fmulx.v2f64(<2 x double> %a, <2 x double> %shuffle)4221 ret <2 x double> %vmulx2.i4222}4223 4224define <4 x float> @optimize_dup(<4 x float> %a, <4 x float> %b, <4 x float> %c, <4 x float> %v) {4225; CHECK-LABEL: optimize_dup:4226; CHECK: // %bb.0: // %entry4227; CHECK-NEXT: fmla v0.4s, v1.4s, v3.s[3]4228; CHECK-NEXT: fmls v0.4s, v2.4s, v3.s[3]4229; CHECK-NEXT: ret4230entry:4231 %lane1 = shufflevector <4 x float> %v, <4 x float> undef, <4 x i32> <i32 3, i32 3, i32 3, i32 3>4232 %0 = tail call <4 x float> @llvm.fma.v4f32(<4 x float> %lane1, <4 x float> %b, <4 x float> %a)4233 %lane2 = shufflevector <4 x float> %v, <4 x float> undef, <4 x i32> <i32 3, i32 3, i32 3, i32 3>4234 %1 = fmul <4 x float> %lane2, %c4235 %s = fsub <4 x float> %0, %14236 ret <4 x float> %s4237}4238 4239define <4 x float> @no_optimize_dup(<4 x float> %a, <4 x float> %b, <4 x float> %c, <4 x float> %v) {4240; CHECK-LABEL: no_optimize_dup:4241; CHECK: // %bb.0: // %entry4242; CHECK-NEXT: fmla v0.4s, v1.4s, v3.s[3]4243; CHECK-NEXT: fmls v0.4s, v2.4s, v3.s[1]4244; CHECK-NEXT: ret4245entry:4246 %lane1 = shufflevector <4 x float> %v, <4 x float> undef, <4 x i32> <i32 3, i32 3, i32 3, i32 3>4247 %0 = tail call <4 x float> @llvm.fma.v4f32(<4 x float> %lane1, <4 x float> %b, <4 x float> %a)4248 %lane2 = shufflevector <4 x float> %v, <4 x float> undef, <4 x i32> <i32 1, i32 1, i32 1, i32 1>4249 %1 = fmul <4 x float> %lane2, %c4250 %s = fsub <4 x float> %0, %14251 ret <4 x float> %s4252}4253 4254define <2 x float> @test_vfma_lane_simdinstr_opt_pass_caching_a57(<2 x float> %a, <2 x float> %b, <2 x float> %v) "target-cpu"="cortex-a57" {4255; CHECK-LABEL: test_vfma_lane_simdinstr_opt_pass_caching_a57:4256; CHECK: // %bb.0: // %entry4257; CHECK-NEXT: // kill: def $d2 killed $d2 def $q24258; CHECK-NEXT: fmla v0.2s, v1.2s, v2.s[1]4259; CHECK-NEXT: ret4260entry:4261 %lane = shufflevector <2 x float> %v, <2 x float> undef, <2 x i32> <i32 1, i32 1>4262 %0 = tail call <2 x float> @llvm.fma.v2f32(<2 x float> %lane, <2 x float> %b, <2 x float> %a)4263 ret <2 x float> %04264}4265 4266define <2 x float> @test_vfma_lane_simdinstr_opt_pass_caching_m3(<2 x float> %a, <2 x float> %b, <2 x float> %v) "target-cpu"="exynos-m3" {4267; CHECK-LABEL: test_vfma_lane_simdinstr_opt_pass_caching_m3:4268; CHECK: // %bb.0: // %entry4269; CHECK-NEXT: // kill: def $d2 killed $d2 def $q24270; CHECK-NEXT: fmla v0.2s, v1.2s, v2.s[1]4271; CHECK-NEXT: ret4272entry:4273 %lane = shufflevector <2 x float> %v, <2 x float> undef, <2 x i32> <i32 1, i32 1>4274 %0 = tail call <2 x float> @llvm.fma.v2f32(<2 x float> %lane, <2 x float> %b, <2 x float> %a)4275 ret <2 x float> %04276}4277