3543 lines · plain
1; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py2; RUN: llc -mtriple=aarch64-none-elf -mattr=+aes < %s | FileCheck %s --check-prefixes=CHECK,CHECK-SD3; RUN: llc -mtriple=aarch64-none-elf -mattr=+aes -global-isel -global-isel-abort=2 2>&1 < %s | FileCheck %s --check-prefixes=CHECK,CHECK-GI4 5; CHECK-GI: warning: Instruction selection used fallback path for sqdmulh_1s6; CHECK-GI-NEXT: warning: Instruction selection used fallback path for fmls_2s7; CHECK-GI-NEXT: warning: Instruction selection used fallback path for fmls_4s8; CHECK-GI-NEXT: warning: Instruction selection used fallback path for fmls_2d9; CHECK-GI-NEXT: warning: Instruction selection used fallback path for fmls_commuted_neg_2s10; CHECK-GI-NEXT: warning: Instruction selection used fallback path for fmls_commuted_neg_4s11; CHECK-GI-NEXT: warning: Instruction selection used fallback path for fmls_commuted_neg_2d12; CHECK-GI-NEXT: warning: Instruction selection used fallback path for fmls_indexed_2s13; CHECK-GI-NEXT: warning: Instruction selection used fallback path for fmls_indexed_4s14; CHECK-GI-NEXT: warning: Instruction selection used fallback path for fmls_indexed_2d15; CHECK-GI-NEXT: warning: Instruction selection used fallback path for fmls_indexed_2s_strict16; CHECK-GI-NEXT: warning: Instruction selection used fallback path for fmls_indexed_4s_strict17; CHECK-GI-NEXT: warning: Instruction selection used fallback path for fmls_indexed_2d_strict18; CHECK-GI-NEXT: warning: Instruction selection used fallback path for fmla_indexed_scalar_2s_strict19; CHECK-GI-NEXT: warning: Instruction selection used fallback path for fmla_indexed_scalar_4s_strict20; CHECK-GI-NEXT: warning: Instruction selection used fallback path for fmla_indexed_scalar_2d_strict21; CHECK-GI-NEXT: warning: Instruction selection used fallback path for sqdmulh_lane_1s22; CHECK-GI-NEXT: warning: Instruction selection used fallback path for sqdmlal_lane_1d23; CHECK-GI-NEXT: warning: Instruction selection used fallback path for sqdmlsl_lane_1d24; CHECK-GI-NEXT: warning: Instruction selection used fallback path for scalar_fmls_from_extract_v4f3225; CHECK-GI-NEXT: warning: Instruction selection used fallback path for scalar_fmls_from_extract_v2f3226; CHECK-GI-NEXT: warning: Instruction selection used fallback path for scalar_fmls_from_extract_v2f6427; CHECK-GI-NEXT: warning: Instruction selection used fallback path for fmls_with_fneg_before_extract_v2f3228; CHECK-GI-NEXT: warning: Instruction selection used fallback path for fmls_with_fneg_before_extract_v2f32_129; CHECK-GI-NEXT: warning: Instruction selection used fallback path for fmls_with_fneg_before_extract_v4f3230; CHECK-GI-NEXT: warning: Instruction selection used fallback path for fmls_with_fneg_before_extract_v4f32_131; CHECK-GI-NEXT: warning: Instruction selection used fallback path for fmls_with_fneg_before_extract_v2f6432; CHECK-GI-NEXT: warning: Instruction selection used fallback path for sqdmlal_d33; CHECK-GI-NEXT: warning: Instruction selection used fallback path for sqdmlsl_d34 35define <8 x i16> @smull8h(ptr %A, ptr %B) nounwind {36; CHECK-LABEL: smull8h:37; CHECK: // %bb.0:38; CHECK-NEXT: ldr d0, [x0]39; CHECK-NEXT: ldr d1, [x1]40; CHECK-NEXT: smull v0.8h, v0.8b, v1.8b41; CHECK-NEXT: ret42 %tmp1 = load <8 x i8>, ptr %A43 %tmp2 = load <8 x i8>, ptr %B44 %tmp3 = call <8 x i16> @llvm.aarch64.neon.smull.v8i16(<8 x i8> %tmp1, <8 x i8> %tmp2)45 ret <8 x i16> %tmp346}47 48define <4 x i32> @smull4s(ptr %A, ptr %B) nounwind {49; CHECK-LABEL: smull4s:50; CHECK: // %bb.0:51; CHECK-NEXT: ldr d0, [x0]52; CHECK-NEXT: ldr d1, [x1]53; CHECK-NEXT: smull v0.4s, v0.4h, v1.4h54; CHECK-NEXT: ret55 %tmp1 = load <4 x i16>, ptr %A56 %tmp2 = load <4 x i16>, ptr %B57 %tmp3 = call <4 x i32> @llvm.aarch64.neon.smull.v4i32(<4 x i16> %tmp1, <4 x i16> %tmp2)58 ret <4 x i32> %tmp359}60 61define <2 x i64> @smull2d(ptr %A, ptr %B) nounwind {62; CHECK-LABEL: smull2d:63; CHECK: // %bb.0:64; CHECK-NEXT: ldr d0, [x0]65; CHECK-NEXT: ldr d1, [x1]66; CHECK-NEXT: smull v0.2d, v0.2s, v1.2s67; CHECK-NEXT: ret68 %tmp1 = load <2 x i32>, ptr %A69 %tmp2 = load <2 x i32>, ptr %B70 %tmp3 = call <2 x i64> @llvm.aarch64.neon.smull.v2i64(<2 x i32> %tmp1, <2 x i32> %tmp2)71 ret <2 x i64> %tmp372}73 74define void @commutable_smull(<2 x i32> %A, <2 x i32> %B, ptr %C) {75; CHECK-LABEL: commutable_smull:76; CHECK: // %bb.0:77; CHECK-NEXT: smull v0.2d, v0.2s, v1.2s78; CHECK-NEXT: stp q0, q0, [x0]79; CHECK-NEXT: ret80 %1 = call <2 x i64> @llvm.aarch64.neon.smull.v2i64(<2 x i32> %A, <2 x i32> %B)81 %2 = call <2 x i64> @llvm.aarch64.neon.smull.v2i64(<2 x i32> %B, <2 x i32> %A)82 store <2 x i64> %1, ptr %C83 %3 = getelementptr i8, ptr %C, i64 1684 store <2 x i64> %2, ptr %385 ret void86}87 88declare <8 x i16> @llvm.aarch64.neon.smull.v8i16(<8 x i8>, <8 x i8>) nounwind readnone89declare <4 x i32> @llvm.aarch64.neon.smull.v4i32(<4 x i16>, <4 x i16>) nounwind readnone90declare <2 x i64> @llvm.aarch64.neon.smull.v2i64(<2 x i32>, <2 x i32>) nounwind readnone91 92define <8 x i16> @umull8h(ptr %A, ptr %B) nounwind {93; CHECK-LABEL: umull8h:94; CHECK: // %bb.0:95; CHECK-NEXT: ldr d0, [x0]96; CHECK-NEXT: ldr d1, [x1]97; CHECK-NEXT: umull v0.8h, v0.8b, v1.8b98; CHECK-NEXT: ret99 %tmp1 = load <8 x i8>, ptr %A100 %tmp2 = load <8 x i8>, ptr %B101 %tmp3 = call <8 x i16> @llvm.aarch64.neon.umull.v8i16(<8 x i8> %tmp1, <8 x i8> %tmp2)102 ret <8 x i16> %tmp3103}104 105define <4 x i32> @umull4s(ptr %A, ptr %B) nounwind {106; CHECK-LABEL: umull4s:107; CHECK: // %bb.0:108; CHECK-NEXT: ldr d0, [x0]109; CHECK-NEXT: ldr d1, [x1]110; CHECK-NEXT: umull v0.4s, v0.4h, v1.4h111; CHECK-NEXT: ret112 %tmp1 = load <4 x i16>, ptr %A113 %tmp2 = load <4 x i16>, ptr %B114 %tmp3 = call <4 x i32> @llvm.aarch64.neon.umull.v4i32(<4 x i16> %tmp1, <4 x i16> %tmp2)115 ret <4 x i32> %tmp3116}117 118define <2 x i64> @umull2d(ptr %A, ptr %B) nounwind {119; CHECK-LABEL: umull2d:120; CHECK: // %bb.0:121; CHECK-NEXT: ldr d0, [x0]122; CHECK-NEXT: ldr d1, [x1]123; CHECK-NEXT: umull v0.2d, v0.2s, v1.2s124; CHECK-NEXT: ret125 %tmp1 = load <2 x i32>, ptr %A126 %tmp2 = load <2 x i32>, ptr %B127 %tmp3 = call <2 x i64> @llvm.aarch64.neon.umull.v2i64(<2 x i32> %tmp1, <2 x i32> %tmp2)128 ret <2 x i64> %tmp3129}130 131define void @commutable_umull(<2 x i32> %A, <2 x i32> %B, ptr %C) {132; CHECK-LABEL: commutable_umull:133; CHECK: // %bb.0:134; CHECK-NEXT: umull v0.2d, v0.2s, v1.2s135; CHECK-NEXT: stp q0, q0, [x0]136; CHECK-NEXT: ret137 %1 = call <2 x i64> @llvm.aarch64.neon.umull.v2i64(<2 x i32> %A, <2 x i32> %B)138 %2 = call <2 x i64> @llvm.aarch64.neon.umull.v2i64(<2 x i32> %B, <2 x i32> %A)139 store <2 x i64> %1, ptr %C140 %3 = getelementptr i8, ptr %C, i64 16141 store <2 x i64> %2, ptr %3142 ret void143}144 145declare <8 x i16> @llvm.aarch64.neon.umull.v8i16(<8 x i8>, <8 x i8>) nounwind readnone146declare <4 x i32> @llvm.aarch64.neon.umull.v4i32(<4 x i16>, <4 x i16>) nounwind readnone147declare <2 x i64> @llvm.aarch64.neon.umull.v2i64(<2 x i32>, <2 x i32>) nounwind readnone148 149define <4 x i32> @sqdmull4s(ptr %A, ptr %B) nounwind {150; CHECK-LABEL: sqdmull4s:151; CHECK: // %bb.0:152; CHECK-NEXT: ldr d0, [x0]153; CHECK-NEXT: ldr d1, [x1]154; CHECK-NEXT: sqdmull v0.4s, v0.4h, v1.4h155; CHECK-NEXT: ret156 %tmp1 = load <4 x i16>, ptr %A157 %tmp2 = load <4 x i16>, ptr %B158 %tmp3 = call <4 x i32> @llvm.aarch64.neon.sqdmull.v4i32(<4 x i16> %tmp1, <4 x i16> %tmp2)159 ret <4 x i32> %tmp3160}161 162define <2 x i64> @sqdmull2d(ptr %A, ptr %B) nounwind {163; CHECK-LABEL: sqdmull2d:164; CHECK: // %bb.0:165; CHECK-NEXT: ldr d0, [x0]166; CHECK-NEXT: ldr d1, [x1]167; CHECK-NEXT: sqdmull v0.2d, v0.2s, v1.2s168; CHECK-NEXT: ret169 %tmp1 = load <2 x i32>, ptr %A170 %tmp2 = load <2 x i32>, ptr %B171 %tmp3 = call <2 x i64> @llvm.aarch64.neon.sqdmull.v2i64(<2 x i32> %tmp1, <2 x i32> %tmp2)172 ret <2 x i64> %tmp3173}174 175define <4 x i32> @sqdmull2_4s(ptr %A, ptr %B) nounwind {176; CHECK-SD-LABEL: sqdmull2_4s:177; CHECK-SD: // %bb.0:178; CHECK-SD-NEXT: ldr d0, [x0, #8]179; CHECK-SD-NEXT: ldr d1, [x1, #8]180; CHECK-SD-NEXT: sqdmull v0.4s, v0.4h, v1.4h181; CHECK-SD-NEXT: ret182;183; CHECK-GI-LABEL: sqdmull2_4s:184; CHECK-GI: // %bb.0:185; CHECK-GI-NEXT: ldr q0, [x0]186; CHECK-GI-NEXT: ldr q1, [x1]187; CHECK-GI-NEXT: sqdmull2 v0.4s, v0.8h, v1.8h188; CHECK-GI-NEXT: ret189 %load1 = load <8 x i16>, ptr %A190 %load2 = load <8 x i16>, ptr %B191 %tmp1 = shufflevector <8 x i16> %load1, <8 x i16> undef, <4 x i32> <i32 4, i32 5, i32 6, i32 7>192 %tmp2 = shufflevector <8 x i16> %load2, <8 x i16> undef, <4 x i32> <i32 4, i32 5, i32 6, i32 7>193 %tmp3 = call <4 x i32> @llvm.aarch64.neon.sqdmull.v4i32(<4 x i16> %tmp1, <4 x i16> %tmp2)194 ret <4 x i32> %tmp3195}196 197define <2 x i64> @sqdmull2_2d(ptr %A, ptr %B) nounwind {198; CHECK-SD-LABEL: sqdmull2_2d:199; CHECK-SD: // %bb.0:200; CHECK-SD-NEXT: ldr d0, [x0, #8]201; CHECK-SD-NEXT: ldr d1, [x1, #8]202; CHECK-SD-NEXT: sqdmull v0.2d, v0.2s, v1.2s203; CHECK-SD-NEXT: ret204;205; CHECK-GI-LABEL: sqdmull2_2d:206; CHECK-GI: // %bb.0:207; CHECK-GI-NEXT: ldr q0, [x0]208; CHECK-GI-NEXT: ldr q1, [x1]209; CHECK-GI-NEXT: sqdmull2 v0.2d, v0.4s, v1.4s210; CHECK-GI-NEXT: ret211 %load1 = load <4 x i32>, ptr %A212 %load2 = load <4 x i32>, ptr %B213 %tmp1 = shufflevector <4 x i32> %load1, <4 x i32> undef, <2 x i32> <i32 2, i32 3>214 %tmp2 = shufflevector <4 x i32> %load2, <4 x i32> undef, <2 x i32> <i32 2, i32 3>215 %tmp3 = call <2 x i64> @llvm.aarch64.neon.sqdmull.v2i64(<2 x i32> %tmp1, <2 x i32> %tmp2)216 ret <2 x i64> %tmp3217}218 219 220declare <4 x i32> @llvm.aarch64.neon.sqdmull.v4i32(<4 x i16>, <4 x i16>) nounwind readnone221declare <2 x i64> @llvm.aarch64.neon.sqdmull.v2i64(<2 x i32>, <2 x i32>) nounwind readnone222 223define <8 x i16> @pmull8h(ptr %A, ptr %B) nounwind {224; CHECK-LABEL: pmull8h:225; CHECK: // %bb.0:226; CHECK-NEXT: ldr d0, [x0]227; CHECK-NEXT: ldr d1, [x1]228; CHECK-NEXT: pmull v0.8h, v0.8b, v1.8b229; CHECK-NEXT: ret230 %tmp1 = load <8 x i8>, ptr %A231 %tmp2 = load <8 x i8>, ptr %B232 %tmp3 = call <8 x i16> @llvm.aarch64.neon.pmull.v8i16(<8 x i8> %tmp1, <8 x i8> %tmp2)233 ret <8 x i16> %tmp3234}235 236define void @commutable_pmull8h(<8 x i8> %A, <8 x i8> %B, ptr %C) {237; CHECK-LABEL: commutable_pmull8h:238; CHECK: // %bb.0:239; CHECK-NEXT: pmull v0.8h, v0.8b, v1.8b240; CHECK-NEXT: stp q0, q0, [x0]241; CHECK-NEXT: ret242 %1 = call <8 x i16> @llvm.aarch64.neon.pmull.v8i16(<8 x i8> %A, <8 x i8> %B)243 %2 = call <8 x i16> @llvm.aarch64.neon.pmull.v8i16(<8 x i8> %B, <8 x i8> %A)244 store <8 x i16> %1, ptr %C245 %3 = getelementptr i8, ptr %C, i8 16246 store <8 x i16> %2, ptr %3247 ret void248}249 250declare <8 x i16> @llvm.aarch64.neon.pmull.v8i16(<8 x i8>, <8 x i8>) nounwind readnone251 252define <4 x i16> @sqdmulh_4h(ptr %A, ptr %B) nounwind {253; CHECK-LABEL: sqdmulh_4h:254; CHECK: // %bb.0:255; CHECK-NEXT: ldr d0, [x0]256; CHECK-NEXT: ldr d1, [x1]257; CHECK-NEXT: sqdmulh v0.4h, v0.4h, v1.4h258; CHECK-NEXT: ret259 %tmp1 = load <4 x i16>, ptr %A260 %tmp2 = load <4 x i16>, ptr %B261 %tmp3 = call <4 x i16> @llvm.aarch64.neon.sqdmulh.v4i16(<4 x i16> %tmp1, <4 x i16> %tmp2)262 ret <4 x i16> %tmp3263}264 265define <8 x i16> @sqdmulh_8h(ptr %A, ptr %B) nounwind {266; CHECK-LABEL: sqdmulh_8h:267; CHECK: // %bb.0:268; CHECK-NEXT: ldr q0, [x0]269; CHECK-NEXT: ldr q1, [x1]270; CHECK-NEXT: sqdmulh v0.8h, v0.8h, v1.8h271; CHECK-NEXT: ret272 %tmp1 = load <8 x i16>, ptr %A273 %tmp2 = load <8 x i16>, ptr %B274 %tmp3 = call <8 x i16> @llvm.aarch64.neon.sqdmulh.v8i16(<8 x i16> %tmp1, <8 x i16> %tmp2)275 ret <8 x i16> %tmp3276}277 278define <2 x i32> @sqdmulh_2s(ptr %A, ptr %B) nounwind {279; CHECK-LABEL: sqdmulh_2s:280; CHECK: // %bb.0:281; CHECK-NEXT: ldr d0, [x0]282; CHECK-NEXT: ldr d1, [x1]283; CHECK-NEXT: sqdmulh v0.2s, v0.2s, v1.2s284; CHECK-NEXT: ret285 %tmp1 = load <2 x i32>, ptr %A286 %tmp2 = load <2 x i32>, ptr %B287 %tmp3 = call <2 x i32> @llvm.aarch64.neon.sqdmulh.v2i32(<2 x i32> %tmp1, <2 x i32> %tmp2)288 ret <2 x i32> %tmp3289}290 291define <4 x i32> @sqdmulh_4s(ptr %A, ptr %B) nounwind {292; CHECK-LABEL: sqdmulh_4s:293; CHECK: // %bb.0:294; CHECK-NEXT: ldr q0, [x0]295; CHECK-NEXT: ldr q1, [x1]296; CHECK-NEXT: sqdmulh v0.4s, v0.4s, v1.4s297; CHECK-NEXT: ret298 %tmp1 = load <4 x i32>, ptr %A299 %tmp2 = load <4 x i32>, ptr %B300 %tmp3 = call <4 x i32> @llvm.aarch64.neon.sqdmulh.v4i32(<4 x i32> %tmp1, <4 x i32> %tmp2)301 ret <4 x i32> %tmp3302}303 304define i32 @sqdmulh_1s(ptr %A, ptr %B) nounwind {305; CHECK-LABEL: sqdmulh_1s:306; CHECK: // %bb.0:307; CHECK-NEXT: ldr w8, [x0]308; CHECK-NEXT: ldr w9, [x1]309; CHECK-NEXT: fmov s0, w8310; CHECK-NEXT: fmov s1, w9311; CHECK-NEXT: sqdmulh s0, s0, s1312; CHECK-NEXT: fmov w0, s0313; CHECK-NEXT: ret314 %tmp1 = load i32, ptr %A315 %tmp2 = load i32, ptr %B316 %tmp3 = call i32 @llvm.aarch64.neon.sqdmulh.i32(i32 %tmp1, i32 %tmp2)317 ret i32 %tmp3318}319 320declare <4 x i16> @llvm.aarch64.neon.sqdmulh.v4i16(<4 x i16>, <4 x i16>) nounwind readnone321declare <8 x i16> @llvm.aarch64.neon.sqdmulh.v8i16(<8 x i16>, <8 x i16>) nounwind readnone322declare <2 x i32> @llvm.aarch64.neon.sqdmulh.v2i32(<2 x i32>, <2 x i32>) nounwind readnone323declare <4 x i32> @llvm.aarch64.neon.sqdmulh.v4i32(<4 x i32>, <4 x i32>) nounwind readnone324declare i32 @llvm.aarch64.neon.sqdmulh.i32(i32, i32) nounwind readnone325 326define <4 x i16> @sqrdmulh_4h(ptr %A, ptr %B) nounwind {327; CHECK-LABEL: sqrdmulh_4h:328; CHECK: // %bb.0:329; CHECK-NEXT: ldr d0, [x0]330; CHECK-NEXT: ldr d1, [x1]331; CHECK-NEXT: sqrdmulh v0.4h, v0.4h, v1.4h332; CHECK-NEXT: ret333 %tmp1 = load <4 x i16>, ptr %A334 %tmp2 = load <4 x i16>, ptr %B335 %tmp3 = call <4 x i16> @llvm.aarch64.neon.sqrdmulh.v4i16(<4 x i16> %tmp1, <4 x i16> %tmp2)336 ret <4 x i16> %tmp3337}338 339define <8 x i16> @sqrdmulh_8h(ptr %A, ptr %B) nounwind {340; CHECK-LABEL: sqrdmulh_8h:341; CHECK: // %bb.0:342; CHECK-NEXT: ldr q0, [x0]343; CHECK-NEXT: ldr q1, [x1]344; CHECK-NEXT: sqrdmulh v0.8h, v0.8h, v1.8h345; CHECK-NEXT: ret346 %tmp1 = load <8 x i16>, ptr %A347 %tmp2 = load <8 x i16>, ptr %B348 %tmp3 = call <8 x i16> @llvm.aarch64.neon.sqrdmulh.v8i16(<8 x i16> %tmp1, <8 x i16> %tmp2)349 ret <8 x i16> %tmp3350}351 352define <2 x i32> @sqrdmulh_2s(ptr %A, ptr %B) nounwind {353; CHECK-LABEL: sqrdmulh_2s:354; CHECK: // %bb.0:355; CHECK-NEXT: ldr d0, [x0]356; CHECK-NEXT: ldr d1, [x1]357; CHECK-NEXT: sqrdmulh v0.2s, v0.2s, v1.2s358; CHECK-NEXT: ret359 %tmp1 = load <2 x i32>, ptr %A360 %tmp2 = load <2 x i32>, ptr %B361 %tmp3 = call <2 x i32> @llvm.aarch64.neon.sqrdmulh.v2i32(<2 x i32> %tmp1, <2 x i32> %tmp2)362 ret <2 x i32> %tmp3363}364 365define <4 x i32> @sqrdmulh_4s(ptr %A, ptr %B) nounwind {366; CHECK-LABEL: sqrdmulh_4s:367; CHECK: // %bb.0:368; CHECK-NEXT: ldr q0, [x0]369; CHECK-NEXT: ldr q1, [x1]370; CHECK-NEXT: sqrdmulh v0.4s, v0.4s, v1.4s371; CHECK-NEXT: ret372 %tmp1 = load <4 x i32>, ptr %A373 %tmp2 = load <4 x i32>, ptr %B374 %tmp3 = call <4 x i32> @llvm.aarch64.neon.sqrdmulh.v4i32(<4 x i32> %tmp1, <4 x i32> %tmp2)375 ret <4 x i32> %tmp3376}377 378define i32 @sqrdmulh_1s(ptr %A, ptr %B) nounwind {379; CHECK-SD-LABEL: sqrdmulh_1s:380; CHECK-SD: // %bb.0:381; CHECK-SD-NEXT: ldr w8, [x0]382; CHECK-SD-NEXT: ldr w9, [x1]383; CHECK-SD-NEXT: fmov s0, w8384; CHECK-SD-NEXT: fmov s1, w9385; CHECK-SD-NEXT: sqrdmulh s0, s0, s1386; CHECK-SD-NEXT: fmov w0, s0387; CHECK-SD-NEXT: ret388;389; CHECK-GI-LABEL: sqrdmulh_1s:390; CHECK-GI: // %bb.0:391; CHECK-GI-NEXT: ldr s0, [x0]392; CHECK-GI-NEXT: ldr s1, [x1]393; CHECK-GI-NEXT: sqrdmulh s0, s0, s1394; CHECK-GI-NEXT: fmov w0, s0395; CHECK-GI-NEXT: ret396 %tmp1 = load i32, ptr %A397 %tmp2 = load i32, ptr %B398 %tmp3 = call i32 @llvm.aarch64.neon.sqrdmulh.i32(i32 %tmp1, i32 %tmp2)399 ret i32 %tmp3400}401 402declare <4 x i16> @llvm.aarch64.neon.sqrdmulh.v4i16(<4 x i16>, <4 x i16>) nounwind readnone403declare <8 x i16> @llvm.aarch64.neon.sqrdmulh.v8i16(<8 x i16>, <8 x i16>) nounwind readnone404declare <2 x i32> @llvm.aarch64.neon.sqrdmulh.v2i32(<2 x i32>, <2 x i32>) nounwind readnone405declare <4 x i32> @llvm.aarch64.neon.sqrdmulh.v4i32(<4 x i32>, <4 x i32>) nounwind readnone406declare i32 @llvm.aarch64.neon.sqrdmulh.i32(i32, i32) nounwind readnone407 408define <2 x float> @fmulx_2s(ptr %A, ptr %B) nounwind {409; CHECK-LABEL: fmulx_2s:410; CHECK: // %bb.0:411; CHECK-NEXT: ldr d0, [x0]412; CHECK-NEXT: ldr d1, [x1]413; CHECK-NEXT: fmulx v0.2s, v0.2s, v1.2s414; CHECK-NEXT: ret415 %tmp1 = load <2 x float>, ptr %A416 %tmp2 = load <2 x float>, ptr %B417 %tmp3 = call <2 x float> @llvm.aarch64.neon.fmulx.v2f32(<2 x float> %tmp1, <2 x float> %tmp2)418 ret <2 x float> %tmp3419}420 421define <4 x float> @fmulx_4s(ptr %A, ptr %B) nounwind {422; CHECK-LABEL: fmulx_4s:423; CHECK: // %bb.0:424; CHECK-NEXT: ldr q0, [x0]425; CHECK-NEXT: ldr q1, [x1]426; CHECK-NEXT: fmulx v0.4s, v0.4s, v1.4s427; CHECK-NEXT: ret428 %tmp1 = load <4 x float>, ptr %A429 %tmp2 = load <4 x float>, ptr %B430 %tmp3 = call <4 x float> @llvm.aarch64.neon.fmulx.v4f32(<4 x float> %tmp1, <4 x float> %tmp2)431 ret <4 x float> %tmp3432}433 434define <2 x double> @fmulx_2d(ptr %A, ptr %B) nounwind {435; CHECK-LABEL: fmulx_2d:436; CHECK: // %bb.0:437; CHECK-NEXT: ldr q0, [x0]438; CHECK-NEXT: ldr q1, [x1]439; CHECK-NEXT: fmulx v0.2d, v0.2d, v1.2d440; CHECK-NEXT: ret441 %tmp1 = load <2 x double>, ptr %A442 %tmp2 = load <2 x double>, ptr %B443 %tmp3 = call <2 x double> @llvm.aarch64.neon.fmulx.v2f64(<2 x double> %tmp1, <2 x double> %tmp2)444 ret <2 x double> %tmp3445}446 447declare <2 x float> @llvm.aarch64.neon.fmulx.v2f32(<2 x float>, <2 x float>) nounwind readnone448declare <4 x float> @llvm.aarch64.neon.fmulx.v4f32(<4 x float>, <4 x float>) nounwind readnone449declare <2 x double> @llvm.aarch64.neon.fmulx.v2f64(<2 x double>, <2 x double>) nounwind readnone450 451define <4 x i32> @smlal4s(ptr %A, ptr %B, ptr %C) nounwind {452; CHECK-LABEL: smlal4s:453; CHECK: // %bb.0:454; CHECK-NEXT: ldr d1, [x0]455; CHECK-NEXT: ldr d2, [x1]456; CHECK-NEXT: ldr q0, [x2]457; CHECK-NEXT: smlal v0.4s, v1.4h, v2.4h458; CHECK-NEXT: ret459 %tmp1 = load <4 x i16>, ptr %A460 %tmp2 = load <4 x i16>, ptr %B461 %tmp3 = load <4 x i32>, ptr %C462 %tmp4 = call <4 x i32> @llvm.aarch64.neon.smull.v4i32(<4 x i16> %tmp1, <4 x i16> %tmp2)463 %tmp5 = add <4 x i32> %tmp3, %tmp4464 ret <4 x i32> %tmp5465}466 467define <2 x i64> @smlal2d(ptr %A, ptr %B, ptr %C) nounwind {468; CHECK-LABEL: smlal2d:469; CHECK: // %bb.0:470; CHECK-NEXT: ldr d1, [x0]471; CHECK-NEXT: ldr d2, [x1]472; CHECK-NEXT: ldr q0, [x2]473; CHECK-NEXT: smlal v0.2d, v1.2s, v2.2s474; CHECK-NEXT: ret475 %tmp1 = load <2 x i32>, ptr %A476 %tmp2 = load <2 x i32>, ptr %B477 %tmp3 = load <2 x i64>, ptr %C478 %tmp4 = call <2 x i64> @llvm.aarch64.neon.smull.v2i64(<2 x i32> %tmp1, <2 x i32> %tmp2)479 %tmp5 = add <2 x i64> %tmp3, %tmp4480 ret <2 x i64> %tmp5481}482 483define void @smlal8h_chain_with_constant(ptr %dst, <8 x i8> %v1, <8 x i8> %v2, <8 x i8> %v3) {484; CHECK-SD-LABEL: smlal8h_chain_with_constant:485; CHECK-SD: // %bb.0:486; CHECK-SD-NEXT: movi v3.16b, #1487; CHECK-SD-NEXT: smlal v3.8h, v0.8b, v2.8b488; CHECK-SD-NEXT: mvn v0.8b, v2.8b489; CHECK-SD-NEXT: smlal v3.8h, v1.8b, v0.8b490; CHECK-SD-NEXT: str q3, [x0]491; CHECK-SD-NEXT: ret492;493; CHECK-GI-LABEL: smlal8h_chain_with_constant:494; CHECK-GI: // %bb.0:495; CHECK-GI-NEXT: mvn v3.8b, v2.8b496; CHECK-GI-NEXT: smull v1.8h, v1.8b, v3.8b497; CHECK-GI-NEXT: movi v3.16b, #1498; CHECK-GI-NEXT: smlal v1.8h, v0.8b, v2.8b499; CHECK-GI-NEXT: add v0.8h, v1.8h, v3.8h500; CHECK-GI-NEXT: str q0, [x0]501; CHECK-GI-NEXT: ret502 %xor = xor <8 x i8> %v3, <i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1>503 %smull.1 = tail call <8 x i16> @llvm.aarch64.neon.smull.v8i16(<8 x i8> %v1, <8 x i8> %v3)504 %add.1 = add <8 x i16> %smull.1, <i16 257, i16 257, i16 257, i16 257, i16 257, i16 257, i16 257, i16 257>505 %smull.2 = tail call <8 x i16> @llvm.aarch64.neon.smull.v8i16(<8 x i8> %v2, <8 x i8> %xor)506 %add.2 = add <8 x i16> %add.1, %smull.2507 store <8 x i16> %add.2, ptr %dst508 ret void509}510 511define void @smlal2d_chain_with_constant(ptr %dst, <2 x i32> %v1, <2 x i32> %v2, <2 x i32> %v3) {512; CHECK-SD-LABEL: smlal2d_chain_with_constant:513; CHECK-SD: // %bb.0:514; CHECK-SD-NEXT: mov w8, #257 // =0x101515; CHECK-SD-NEXT: dup v3.2d, x8516; CHECK-SD-NEXT: smlal v3.2d, v0.2s, v2.2s517; CHECK-SD-NEXT: mvn v0.8b, v2.8b518; CHECK-SD-NEXT: smlal v3.2d, v1.2s, v0.2s519; CHECK-SD-NEXT: str q3, [x0]520; CHECK-SD-NEXT: ret521;522; CHECK-GI-LABEL: smlal2d_chain_with_constant:523; CHECK-GI: // %bb.0:524; CHECK-GI-NEXT: mvn v3.8b, v2.8b525; CHECK-GI-NEXT: adrp x8, .LCPI30_0526; CHECK-GI-NEXT: smull v1.2d, v1.2s, v3.2s527; CHECK-GI-NEXT: smlal v1.2d, v0.2s, v2.2s528; CHECK-GI-NEXT: ldr q0, [x8, :lo12:.LCPI30_0]529; CHECK-GI-NEXT: add v0.2d, v1.2d, v0.2d530; CHECK-GI-NEXT: str q0, [x0]531; CHECK-GI-NEXT: ret532 %xor = xor <2 x i32> %v3, <i32 -1, i32 -1>533 %smull.1 = tail call <2 x i64> @llvm.aarch64.neon.smull.v2i64(<2 x i32> %v1, <2 x i32> %v3)534 %add.1 = add <2 x i64> %smull.1, <i64 257, i64 257>535 %smull.2 = tail call <2 x i64> @llvm.aarch64.neon.smull.v2i64(<2 x i32> %v2, <2 x i32> %xor)536 %add.2 = add <2 x i64> %add.1, %smull.2537 store <2 x i64> %add.2, ptr %dst538 ret void539}540 541define <4 x i32> @smlsl4s(ptr %A, ptr %B, ptr %C) nounwind {542; CHECK-LABEL: smlsl4s:543; CHECK: // %bb.0:544; CHECK-NEXT: ldr d1, [x0]545; CHECK-NEXT: ldr d2, [x1]546; CHECK-NEXT: ldr q0, [x2]547; CHECK-NEXT: smlsl v0.4s, v1.4h, v2.4h548; CHECK-NEXT: ret549 %tmp1 = load <4 x i16>, ptr %A550 %tmp2 = load <4 x i16>, ptr %B551 %tmp3 = load <4 x i32>, ptr %C552 %tmp4 = call <4 x i32> @llvm.aarch64.neon.smull.v4i32(<4 x i16> %tmp1, <4 x i16> %tmp2)553 %tmp5 = sub <4 x i32> %tmp3, %tmp4554 ret <4 x i32> %tmp5555}556 557define <2 x i64> @smlsl2d(ptr %A, ptr %B, ptr %C) nounwind {558; CHECK-LABEL: smlsl2d:559; CHECK: // %bb.0:560; CHECK-NEXT: ldr d1, [x0]561; CHECK-NEXT: ldr d2, [x1]562; CHECK-NEXT: ldr q0, [x2]563; CHECK-NEXT: smlsl v0.2d, v1.2s, v2.2s564; CHECK-NEXT: ret565 %tmp1 = load <2 x i32>, ptr %A566 %tmp2 = load <2 x i32>, ptr %B567 %tmp3 = load <2 x i64>, ptr %C568 %tmp4 = call <2 x i64> @llvm.aarch64.neon.smull.v2i64(<2 x i32> %tmp1, <2 x i32> %tmp2)569 %tmp5 = sub <2 x i64> %tmp3, %tmp4570 ret <2 x i64> %tmp5571}572 573define void @smlsl8h_chain_with_constant(ptr %dst, <8 x i8> %v1, <8 x i8> %v2, <8 x i8> %v3) {574; CHECK-LABEL: smlsl8h_chain_with_constant:575; CHECK: // %bb.0:576; CHECK-NEXT: movi v3.16b, #1577; CHECK-NEXT: smlsl v3.8h, v0.8b, v2.8b578; CHECK-NEXT: mvn v0.8b, v2.8b579; CHECK-NEXT: smlsl v3.8h, v1.8b, v0.8b580; CHECK-NEXT: str q3, [x0]581; CHECK-NEXT: ret582 %xor = xor <8 x i8> %v3, <i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1>583 %smull.1 = tail call <8 x i16> @llvm.aarch64.neon.smull.v8i16(<8 x i8> %v1, <8 x i8> %v3)584 %sub.1 = sub <8 x i16> <i16 257, i16 257, i16 257, i16 257, i16 257, i16 257, i16 257, i16 257>, %smull.1585 %smull.2 = tail call <8 x i16> @llvm.aarch64.neon.smull.v8i16(<8 x i8> %v2, <8 x i8> %xor)586 %sub.2 = sub <8 x i16> %sub.1, %smull.2587 store <8 x i16> %sub.2, ptr %dst588 ret void589}590 591define void @smlsl2d_chain_with_constant(ptr %dst, <2 x i32> %v1, <2 x i32> %v2, <2 x i32> %v3) {592; CHECK-SD-LABEL: smlsl2d_chain_with_constant:593; CHECK-SD: // %bb.0:594; CHECK-SD-NEXT: mov w8, #257 // =0x101595; CHECK-SD-NEXT: dup v3.2d, x8596; CHECK-SD-NEXT: smlsl v3.2d, v0.2s, v2.2s597; CHECK-SD-NEXT: mvn v0.8b, v2.8b598; CHECK-SD-NEXT: smlsl v3.2d, v1.2s, v0.2s599; CHECK-SD-NEXT: str q3, [x0]600; CHECK-SD-NEXT: ret601;602; CHECK-GI-LABEL: smlsl2d_chain_with_constant:603; CHECK-GI: // %bb.0:604; CHECK-GI-NEXT: adrp x8, .LCPI34_0605; CHECK-GI-NEXT: ldr q3, [x8, :lo12:.LCPI34_0]606; CHECK-GI-NEXT: smlsl v3.2d, v0.2s, v2.2s607; CHECK-GI-NEXT: mvn v0.8b, v2.8b608; CHECK-GI-NEXT: smlsl v3.2d, v1.2s, v0.2s609; CHECK-GI-NEXT: str q3, [x0]610; CHECK-GI-NEXT: ret611 %xor = xor <2 x i32> %v3, <i32 -1, i32 -1>612 %smull.1 = tail call <2 x i64> @llvm.aarch64.neon.smull.v2i64(<2 x i32> %v1, <2 x i32> %v3)613 %sub.1 = sub <2 x i64> <i64 257, i64 257>, %smull.1614 %smull.2 = tail call <2 x i64> @llvm.aarch64.neon.smull.v2i64(<2 x i32> %v2, <2 x i32> %xor)615 %sub.2 = sub <2 x i64> %sub.1, %smull.2616 store <2 x i64> %sub.2, ptr %dst617 ret void618}619 620declare <4 x i32> @llvm.aarch64.neon.sqadd.v4i32(<4 x i32>, <4 x i32>)621declare <2 x i64> @llvm.aarch64.neon.sqadd.v2i64(<2 x i64>, <2 x i64>)622declare <4 x i32> @llvm.aarch64.neon.sqsub.v4i32(<4 x i32>, <4 x i32>)623declare <2 x i64> @llvm.aarch64.neon.sqsub.v2i64(<2 x i64>, <2 x i64>)624 625define <4 x i32> @sqdmlal4s(ptr %A, ptr %B, ptr %C) nounwind {626; CHECK-LABEL: sqdmlal4s:627; CHECK: // %bb.0:628; CHECK-NEXT: ldr d1, [x0]629; CHECK-NEXT: ldr d2, [x1]630; CHECK-NEXT: ldr q0, [x2]631; CHECK-NEXT: sqdmlal v0.4s, v1.4h, v2.4h632; CHECK-NEXT: ret633 %tmp1 = load <4 x i16>, ptr %A634 %tmp2 = load <4 x i16>, ptr %B635 %tmp3 = load <4 x i32>, ptr %C636 %tmp4 = call <4 x i32> @llvm.aarch64.neon.sqdmull.v4i32(<4 x i16> %tmp1, <4 x i16> %tmp2)637 %tmp5 = call <4 x i32> @llvm.aarch64.neon.sqadd.v4i32(<4 x i32> %tmp3, <4 x i32> %tmp4)638 ret <4 x i32> %tmp5639}640 641define <2 x i64> @sqdmlal2d(ptr %A, ptr %B, ptr %C) nounwind {642; CHECK-LABEL: sqdmlal2d:643; CHECK: // %bb.0:644; CHECK-NEXT: ldr d1, [x0]645; CHECK-NEXT: ldr d2, [x1]646; CHECK-NEXT: ldr q0, [x2]647; CHECK-NEXT: sqdmlal v0.2d, v1.2s, v2.2s648; CHECK-NEXT: ret649 %tmp1 = load <2 x i32>, ptr %A650 %tmp2 = load <2 x i32>, ptr %B651 %tmp3 = load <2 x i64>, ptr %C652 %tmp4 = call <2 x i64> @llvm.aarch64.neon.sqdmull.v2i64(<2 x i32> %tmp1, <2 x i32> %tmp2)653 %tmp5 = call <2 x i64> @llvm.aarch64.neon.sqadd.v2i64(<2 x i64> %tmp3, <2 x i64> %tmp4)654 ret <2 x i64> %tmp5655}656 657define <4 x i32> @sqdmlal2_4s(ptr %A, ptr %B, ptr %C) nounwind {658; CHECK-SD-LABEL: sqdmlal2_4s:659; CHECK-SD: // %bb.0:660; CHECK-SD-NEXT: ldr q0, [x2]661; CHECK-SD-NEXT: ldr d1, [x0, #8]662; CHECK-SD-NEXT: ldr d2, [x1, #8]663; CHECK-SD-NEXT: sqdmlal v0.4s, v1.4h, v2.4h664; CHECK-SD-NEXT: ret665;666; CHECK-GI-LABEL: sqdmlal2_4s:667; CHECK-GI: // %bb.0:668; CHECK-GI-NEXT: ldr q1, [x0]669; CHECK-GI-NEXT: ldr q2, [x1]670; CHECK-GI-NEXT: ldr q0, [x2]671; CHECK-GI-NEXT: sqdmlal2 v0.4s, v1.8h, v2.8h672; CHECK-GI-NEXT: ret673 %load1 = load <8 x i16>, ptr %A674 %load2 = load <8 x i16>, ptr %B675 %tmp3 = load <4 x i32>, ptr %C676 %tmp1 = shufflevector <8 x i16> %load1, <8 x i16> undef, <4 x i32> <i32 4, i32 5, i32 6, i32 7>677 %tmp2 = shufflevector <8 x i16> %load2, <8 x i16> undef, <4 x i32> <i32 4, i32 5, i32 6, i32 7>678 %tmp4 = call <4 x i32> @llvm.aarch64.neon.sqdmull.v4i32(<4 x i16> %tmp1, <4 x i16> %tmp2)679 %tmp5 = call <4 x i32> @llvm.aarch64.neon.sqadd.v4i32(<4 x i32> %tmp3, <4 x i32> %tmp4)680 ret <4 x i32> %tmp5681}682 683define <2 x i64> @sqdmlal2_2d(ptr %A, ptr %B, ptr %C) nounwind {684; CHECK-SD-LABEL: sqdmlal2_2d:685; CHECK-SD: // %bb.0:686; CHECK-SD-NEXT: ldr q0, [x2]687; CHECK-SD-NEXT: ldr d1, [x0, #8]688; CHECK-SD-NEXT: ldr d2, [x1, #8]689; CHECK-SD-NEXT: sqdmlal v0.2d, v1.2s, v2.2s690; CHECK-SD-NEXT: ret691;692; CHECK-GI-LABEL: sqdmlal2_2d:693; CHECK-GI: // %bb.0:694; CHECK-GI-NEXT: ldr q1, [x0]695; CHECK-GI-NEXT: ldr q2, [x1]696; CHECK-GI-NEXT: ldr q0, [x2]697; CHECK-GI-NEXT: sqdmlal2 v0.2d, v1.4s, v2.4s698; CHECK-GI-NEXT: ret699 %load1 = load <4 x i32>, ptr %A700 %load2 = load <4 x i32>, ptr %B701 %tmp3 = load <2 x i64>, ptr %C702 %tmp1 = shufflevector <4 x i32> %load1, <4 x i32> undef, <2 x i32> <i32 2, i32 3>703 %tmp2 = shufflevector <4 x i32> %load2, <4 x i32> undef, <2 x i32> <i32 2, i32 3>704 %tmp4 = call <2 x i64> @llvm.aarch64.neon.sqdmull.v2i64(<2 x i32> %tmp1, <2 x i32> %tmp2)705 %tmp5 = call <2 x i64> @llvm.aarch64.neon.sqadd.v2i64(<2 x i64> %tmp3, <2 x i64> %tmp4)706 ret <2 x i64> %tmp5707}708 709define <4 x i32> @sqdmlsl4s(ptr %A, ptr %B, ptr %C) nounwind {710; CHECK-LABEL: sqdmlsl4s:711; CHECK: // %bb.0:712; CHECK-NEXT: ldr d1, [x0]713; CHECK-NEXT: ldr d2, [x1]714; CHECK-NEXT: ldr q0, [x2]715; CHECK-NEXT: sqdmlsl v0.4s, v1.4h, v2.4h716; CHECK-NEXT: ret717 %tmp1 = load <4 x i16>, ptr %A718 %tmp2 = load <4 x i16>, ptr %B719 %tmp3 = load <4 x i32>, ptr %C720 %tmp4 = call <4 x i32> @llvm.aarch64.neon.sqdmull.v4i32(<4 x i16> %tmp1, <4 x i16> %tmp2)721 %tmp5 = call <4 x i32> @llvm.aarch64.neon.sqsub.v4i32(<4 x i32> %tmp3, <4 x i32> %tmp4)722 ret <4 x i32> %tmp5723}724 725define <2 x i64> @sqdmlsl2d(ptr %A, ptr %B, ptr %C) nounwind {726; CHECK-LABEL: sqdmlsl2d:727; CHECK: // %bb.0:728; CHECK-NEXT: ldr d1, [x0]729; CHECK-NEXT: ldr d2, [x1]730; CHECK-NEXT: ldr q0, [x2]731; CHECK-NEXT: sqdmlsl v0.2d, v1.2s, v2.2s732; CHECK-NEXT: ret733 %tmp1 = load <2 x i32>, ptr %A734 %tmp2 = load <2 x i32>, ptr %B735 %tmp3 = load <2 x i64>, ptr %C736 %tmp4 = call <2 x i64> @llvm.aarch64.neon.sqdmull.v2i64(<2 x i32> %tmp1, <2 x i32> %tmp2)737 %tmp5 = call <2 x i64> @llvm.aarch64.neon.sqsub.v2i64(<2 x i64> %tmp3, <2 x i64> %tmp4)738 ret <2 x i64> %tmp5739}740 741define <4 x i32> @sqdmlsl2_4s(ptr %A, ptr %B, ptr %C) nounwind {742; CHECK-SD-LABEL: sqdmlsl2_4s:743; CHECK-SD: // %bb.0:744; CHECK-SD-NEXT: ldr q0, [x2]745; CHECK-SD-NEXT: ldr d1, [x0, #8]746; CHECK-SD-NEXT: ldr d2, [x1, #8]747; CHECK-SD-NEXT: sqdmlsl v0.4s, v1.4h, v2.4h748; CHECK-SD-NEXT: ret749;750; CHECK-GI-LABEL: sqdmlsl2_4s:751; CHECK-GI: // %bb.0:752; CHECK-GI-NEXT: ldr q1, [x0]753; CHECK-GI-NEXT: ldr q2, [x1]754; CHECK-GI-NEXT: ldr q0, [x2]755; CHECK-GI-NEXT: sqdmlsl2 v0.4s, v1.8h, v2.8h756; CHECK-GI-NEXT: ret757 %load1 = load <8 x i16>, ptr %A758 %load2 = load <8 x i16>, ptr %B759 %tmp3 = load <4 x i32>, ptr %C760 %tmp1 = shufflevector <8 x i16> %load1, <8 x i16> undef, <4 x i32> <i32 4, i32 5, i32 6, i32 7>761 %tmp2 = shufflevector <8 x i16> %load2, <8 x i16> undef, <4 x i32> <i32 4, i32 5, i32 6, i32 7>762 %tmp4 = call <4 x i32> @llvm.aarch64.neon.sqdmull.v4i32(<4 x i16> %tmp1, <4 x i16> %tmp2)763 %tmp5 = call <4 x i32> @llvm.aarch64.neon.sqsub.v4i32(<4 x i32> %tmp3, <4 x i32> %tmp4)764 ret <4 x i32> %tmp5765}766 767define <2 x i64> @sqdmlsl2_2d(ptr %A, ptr %B, ptr %C) nounwind {768; CHECK-SD-LABEL: sqdmlsl2_2d:769; CHECK-SD: // %bb.0:770; CHECK-SD-NEXT: ldr q0, [x2]771; CHECK-SD-NEXT: ldr d1, [x0, #8]772; CHECK-SD-NEXT: ldr d2, [x1, #8]773; CHECK-SD-NEXT: sqdmlsl v0.2d, v1.2s, v2.2s774; CHECK-SD-NEXT: ret775;776; CHECK-GI-LABEL: sqdmlsl2_2d:777; CHECK-GI: // %bb.0:778; CHECK-GI-NEXT: ldr q1, [x0]779; CHECK-GI-NEXT: ldr q2, [x1]780; CHECK-GI-NEXT: ldr q0, [x2]781; CHECK-GI-NEXT: sqdmlsl2 v0.2d, v1.4s, v2.4s782; CHECK-GI-NEXT: ret783 %load1 = load <4 x i32>, ptr %A784 %load2 = load <4 x i32>, ptr %B785 %tmp3 = load <2 x i64>, ptr %C786 %tmp1 = shufflevector <4 x i32> %load1, <4 x i32> undef, <2 x i32> <i32 2, i32 3>787 %tmp2 = shufflevector <4 x i32> %load2, <4 x i32> undef, <2 x i32> <i32 2, i32 3>788 %tmp4 = call <2 x i64> @llvm.aarch64.neon.sqdmull.v2i64(<2 x i32> %tmp1, <2 x i32> %tmp2)789 %tmp5 = call <2 x i64> @llvm.aarch64.neon.sqsub.v2i64(<2 x i64> %tmp3, <2 x i64> %tmp4)790 ret <2 x i64> %tmp5791}792 793define <4 x i32> @umlal4s(ptr %A, ptr %B, ptr %C) nounwind {794; CHECK-LABEL: umlal4s:795; CHECK: // %bb.0:796; CHECK-NEXT: ldr d1, [x0]797; CHECK-NEXT: ldr d2, [x1]798; CHECK-NEXT: ldr q0, [x2]799; CHECK-NEXT: umlal v0.4s, v1.4h, v2.4h800; CHECK-NEXT: ret801 %tmp1 = load <4 x i16>, ptr %A802 %tmp2 = load <4 x i16>, ptr %B803 %tmp3 = load <4 x i32>, ptr %C804 %tmp4 = call <4 x i32> @llvm.aarch64.neon.umull.v4i32(<4 x i16> %tmp1, <4 x i16> %tmp2)805 %tmp5 = add <4 x i32> %tmp3, %tmp4806 ret <4 x i32> %tmp5807}808 809define <2 x i64> @umlal2d(ptr %A, ptr %B, ptr %C) nounwind {810; CHECK-LABEL: umlal2d:811; CHECK: // %bb.0:812; CHECK-NEXT: ldr d1, [x0]813; CHECK-NEXT: ldr d2, [x1]814; CHECK-NEXT: ldr q0, [x2]815; CHECK-NEXT: umlal v0.2d, v1.2s, v2.2s816; CHECK-NEXT: ret817 %tmp1 = load <2 x i32>, ptr %A818 %tmp2 = load <2 x i32>, ptr %B819 %tmp3 = load <2 x i64>, ptr %C820 %tmp4 = call <2 x i64> @llvm.aarch64.neon.umull.v2i64(<2 x i32> %tmp1, <2 x i32> %tmp2)821 %tmp5 = add <2 x i64> %tmp3, %tmp4822 ret <2 x i64> %tmp5823}824 825define void @umlal8h_chain_with_constant(ptr %dst, <8 x i8> %v1, <8 x i8> %v2, <8 x i8> %v3) {826; CHECK-SD-LABEL: umlal8h_chain_with_constant:827; CHECK-SD: // %bb.0:828; CHECK-SD-NEXT: movi v3.16b, #1829; CHECK-SD-NEXT: umlal v3.8h, v0.8b, v2.8b830; CHECK-SD-NEXT: mvn v0.8b, v2.8b831; CHECK-SD-NEXT: umlal v3.8h, v1.8b, v0.8b832; CHECK-SD-NEXT: str q3, [x0]833; CHECK-SD-NEXT: ret834;835; CHECK-GI-LABEL: umlal8h_chain_with_constant:836; CHECK-GI: // %bb.0:837; CHECK-GI-NEXT: mvn v3.8b, v2.8b838; CHECK-GI-NEXT: umull v1.8h, v1.8b, v3.8b839; CHECK-GI-NEXT: movi v3.16b, #1840; CHECK-GI-NEXT: umlal v1.8h, v0.8b, v2.8b841; CHECK-GI-NEXT: add v0.8h, v1.8h, v3.8h842; CHECK-GI-NEXT: str q0, [x0]843; CHECK-GI-NEXT: ret844 %xor = xor <8 x i8> %v3, <i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1>845 %umull.1 = tail call <8 x i16> @llvm.aarch64.neon.umull.v8i16(<8 x i8> %v1, <8 x i8> %v3)846 %add.1 = add <8 x i16> %umull.1, <i16 257, i16 257, i16 257, i16 257, i16 257, i16 257, i16 257, i16 257>847 %umull.2 = tail call <8 x i16> @llvm.aarch64.neon.umull.v8i16(<8 x i8> %v2, <8 x i8> %xor)848 %add.2 = add <8 x i16> %add.1, %umull.2849 store <8 x i16> %add.2, ptr %dst850 ret void851}852 853define void @umlal2d_chain_with_constant(ptr %dst, <2 x i32> %v1, <2 x i32> %v2, <2 x i32> %v3) {854; CHECK-SD-LABEL: umlal2d_chain_with_constant:855; CHECK-SD: // %bb.0:856; CHECK-SD-NEXT: mov w8, #257 // =0x101857; CHECK-SD-NEXT: dup v3.2d, x8858; CHECK-SD-NEXT: umlal v3.2d, v0.2s, v2.2s859; CHECK-SD-NEXT: mvn v0.8b, v2.8b860; CHECK-SD-NEXT: umlal v3.2d, v1.2s, v0.2s861; CHECK-SD-NEXT: str q3, [x0]862; CHECK-SD-NEXT: ret863;864; CHECK-GI-LABEL: umlal2d_chain_with_constant:865; CHECK-GI: // %bb.0:866; CHECK-GI-NEXT: mvn v3.8b, v2.8b867; CHECK-GI-NEXT: adrp x8, .LCPI46_0868; CHECK-GI-NEXT: umull v1.2d, v1.2s, v3.2s869; CHECK-GI-NEXT: umlal v1.2d, v0.2s, v2.2s870; CHECK-GI-NEXT: ldr q0, [x8, :lo12:.LCPI46_0]871; CHECK-GI-NEXT: add v0.2d, v1.2d, v0.2d872; CHECK-GI-NEXT: str q0, [x0]873; CHECK-GI-NEXT: ret874 %xor = xor <2 x i32> %v3, <i32 -1, i32 -1>875 %umull.1 = tail call <2 x i64> @llvm.aarch64.neon.umull.v2i64(<2 x i32> %v1, <2 x i32> %v3)876 %add.1 = add <2 x i64> %umull.1, <i64 257, i64 257>877 %umull.2 = tail call <2 x i64> @llvm.aarch64.neon.umull.v2i64(<2 x i32> %v2, <2 x i32> %xor)878 %add.2 = add <2 x i64> %add.1, %umull.2879 store <2 x i64> %add.2, ptr %dst880 ret void881}882 883define <4 x i32> @umlsl4s(ptr %A, ptr %B, ptr %C) nounwind {884; CHECK-LABEL: umlsl4s:885; CHECK: // %bb.0:886; CHECK-NEXT: ldr d1, [x0]887; CHECK-NEXT: ldr d2, [x1]888; CHECK-NEXT: ldr q0, [x2]889; CHECK-NEXT: umlsl v0.4s, v1.4h, v2.4h890; CHECK-NEXT: ret891 %tmp1 = load <4 x i16>, ptr %A892 %tmp2 = load <4 x i16>, ptr %B893 %tmp3 = load <4 x i32>, ptr %C894 %tmp4 = call <4 x i32> @llvm.aarch64.neon.umull.v4i32(<4 x i16> %tmp1, <4 x i16> %tmp2)895 %tmp5 = sub <4 x i32> %tmp3, %tmp4896 ret <4 x i32> %tmp5897}898 899define <2 x i64> @umlsl2d(ptr %A, ptr %B, ptr %C) nounwind {900; CHECK-LABEL: umlsl2d:901; CHECK: // %bb.0:902; CHECK-NEXT: ldr d1, [x0]903; CHECK-NEXT: ldr d2, [x1]904; CHECK-NEXT: ldr q0, [x2]905; CHECK-NEXT: umlsl v0.2d, v1.2s, v2.2s906; CHECK-NEXT: ret907 %tmp1 = load <2 x i32>, ptr %A908 %tmp2 = load <2 x i32>, ptr %B909 %tmp3 = load <2 x i64>, ptr %C910 %tmp4 = call <2 x i64> @llvm.aarch64.neon.umull.v2i64(<2 x i32> %tmp1, <2 x i32> %tmp2)911 %tmp5 = sub <2 x i64> %tmp3, %tmp4912 ret <2 x i64> %tmp5913}914 915define void @umlsl8h_chain_with_constant(ptr %dst, <8 x i8> %v1, <8 x i8> %v2, <8 x i8> %v3) {916; CHECK-LABEL: umlsl8h_chain_with_constant:917; CHECK: // %bb.0:918; CHECK-NEXT: movi v3.16b, #1919; CHECK-NEXT: umlsl v3.8h, v0.8b, v2.8b920; CHECK-NEXT: mvn v0.8b, v2.8b921; CHECK-NEXT: umlsl v3.8h, v1.8b, v0.8b922; CHECK-NEXT: str q3, [x0]923; CHECK-NEXT: ret924 %xor = xor <8 x i8> %v3, <i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1>925 %umull.1 = tail call <8 x i16> @llvm.aarch64.neon.umull.v8i16(<8 x i8> %v1, <8 x i8> %v3)926 %add.1 = sub <8 x i16> <i16 257, i16 257, i16 257, i16 257, i16 257, i16 257, i16 257, i16 257>, %umull.1927 %umull.2 = tail call <8 x i16> @llvm.aarch64.neon.umull.v8i16(<8 x i8> %v2, <8 x i8> %xor)928 %add.2 = sub <8 x i16> %add.1, %umull.2929 store <8 x i16> %add.2, ptr %dst930 ret void931}932 933define void @umlsl2d_chain_with_constant(ptr %dst, <2 x i32> %v1, <2 x i32> %v2, <2 x i32> %v3) {934; CHECK-SD-LABEL: umlsl2d_chain_with_constant:935; CHECK-SD: // %bb.0:936; CHECK-SD-NEXT: mov w8, #257 // =0x101937; CHECK-SD-NEXT: dup v3.2d, x8938; CHECK-SD-NEXT: umlsl v3.2d, v0.2s, v2.2s939; CHECK-SD-NEXT: mvn v0.8b, v2.8b940; CHECK-SD-NEXT: umlsl v3.2d, v1.2s, v0.2s941; CHECK-SD-NEXT: str q3, [x0]942; CHECK-SD-NEXT: ret943;944; CHECK-GI-LABEL: umlsl2d_chain_with_constant:945; CHECK-GI: // %bb.0:946; CHECK-GI-NEXT: adrp x8, .LCPI50_0947; CHECK-GI-NEXT: ldr q3, [x8, :lo12:.LCPI50_0]948; CHECK-GI-NEXT: umlsl v3.2d, v0.2s, v2.2s949; CHECK-GI-NEXT: mvn v0.8b, v2.8b950; CHECK-GI-NEXT: umlsl v3.2d, v1.2s, v0.2s951; CHECK-GI-NEXT: str q3, [x0]952; CHECK-GI-NEXT: ret953 %xor = xor <2 x i32> %v3, <i32 -1, i32 -1>954 %umull.1 = tail call <2 x i64> @llvm.aarch64.neon.umull.v2i64(<2 x i32> %v1, <2 x i32> %v3)955 %add.1 = sub <2 x i64> <i64 257, i64 257>, %umull.1956 %umull.2 = tail call <2 x i64> @llvm.aarch64.neon.umull.v2i64(<2 x i32> %v2, <2 x i32> %xor)957 %add.2 = sub <2 x i64> %add.1, %umull.2958 store <2 x i64> %add.2, ptr %dst959 ret void960}961 962define <2 x float> @fmla_2s(ptr %A, ptr %B, ptr %C) nounwind {963; CHECK-LABEL: fmla_2s:964; CHECK: // %bb.0:965; CHECK-NEXT: ldr d1, [x0]966; CHECK-NEXT: ldr d2, [x1]967; CHECK-NEXT: ldr d0, [x2]968; CHECK-NEXT: fmla v0.2s, v2.2s, v1.2s969; CHECK-NEXT: ret970 %tmp1 = load <2 x float>, ptr %A971 %tmp2 = load <2 x float>, ptr %B972 %tmp3 = load <2 x float>, ptr %C973 %tmp4 = call <2 x float> @llvm.fma.v2f32(<2 x float> %tmp1, <2 x float> %tmp2, <2 x float> %tmp3)974 ret <2 x float> %tmp4975}976 977define <4 x float> @fmla_4s(ptr %A, ptr %B, ptr %C) nounwind {978; CHECK-LABEL: fmla_4s:979; CHECK: // %bb.0:980; CHECK-NEXT: ldr q1, [x0]981; CHECK-NEXT: ldr q2, [x1]982; CHECK-NEXT: ldr q0, [x2]983; CHECK-NEXT: fmla v0.4s, v2.4s, v1.4s984; CHECK-NEXT: ret985 %tmp1 = load <4 x float>, ptr %A986 %tmp2 = load <4 x float>, ptr %B987 %tmp3 = load <4 x float>, ptr %C988 %tmp4 = call <4 x float> @llvm.fma.v4f32(<4 x float> %tmp1, <4 x float> %tmp2, <4 x float> %tmp3)989 ret <4 x float> %tmp4990}991 992define <2 x double> @fmla_2d(ptr %A, ptr %B, ptr %C) nounwind {993; CHECK-LABEL: fmla_2d:994; CHECK: // %bb.0:995; CHECK-NEXT: ldr q1, [x0]996; CHECK-NEXT: ldr q2, [x1]997; CHECK-NEXT: ldr q0, [x2]998; CHECK-NEXT: fmla v0.2d, v2.2d, v1.2d999; CHECK-NEXT: ret1000 %tmp1 = load <2 x double>, ptr %A1001 %tmp2 = load <2 x double>, ptr %B1002 %tmp3 = load <2 x double>, ptr %C1003 %tmp4 = call <2 x double> @llvm.fma.v2f64(<2 x double> %tmp1, <2 x double> %tmp2, <2 x double> %tmp3)1004 ret <2 x double> %tmp41005}1006 1007declare <2 x float> @llvm.fma.v2f32(<2 x float>, <2 x float>, <2 x float>) nounwind readnone1008declare <4 x float> @llvm.fma.v4f32(<4 x float>, <4 x float>, <4 x float>) nounwind readnone1009declare <2 x double> @llvm.fma.v2f64(<2 x double>, <2 x double>, <2 x double>) nounwind readnone1010 1011define <2 x float> @fmls_2s(ptr %A, ptr %B, ptr %C) nounwind {1012; CHECK-LABEL: fmls_2s:1013; CHECK: // %bb.0:1014; CHECK-NEXT: ldr d1, [x0]1015; CHECK-NEXT: ldr d2, [x1]1016; CHECK-NEXT: ldr d0, [x2]1017; CHECK-NEXT: fmls v0.2s, v1.2s, v2.2s1018; CHECK-NEXT: ret1019 %tmp1 = load <2 x float>, ptr %A1020 %tmp2 = load <2 x float>, ptr %B1021 %tmp3 = load <2 x float>, ptr %C1022 %tmp4 = fsub <2 x float> <float -0.0, float -0.0>, %tmp21023 %tmp5 = call <2 x float> @llvm.fma.v2f32(<2 x float> %tmp1, <2 x float> %tmp4, <2 x float> %tmp3)1024 ret <2 x float> %tmp51025}1026 1027define <4 x float> @fmls_4s(ptr %A, ptr %B, ptr %C) nounwind {1028; CHECK-LABEL: fmls_4s:1029; CHECK: // %bb.0:1030; CHECK-NEXT: ldr q1, [x0]1031; CHECK-NEXT: ldr q2, [x1]1032; CHECK-NEXT: ldr q0, [x2]1033; CHECK-NEXT: fmls v0.4s, v1.4s, v2.4s1034; CHECK-NEXT: ret1035 %tmp1 = load <4 x float>, ptr %A1036 %tmp2 = load <4 x float>, ptr %B1037 %tmp3 = load <4 x float>, ptr %C1038 %tmp4 = fsub <4 x float> <float -0.0, float -0.0, float -0.0, float -0.0>, %tmp21039 %tmp5 = call <4 x float> @llvm.fma.v4f32(<4 x float> %tmp1, <4 x float> %tmp4, <4 x float> %tmp3)1040 ret <4 x float> %tmp51041}1042 1043define <2 x double> @fmls_2d(ptr %A, ptr %B, ptr %C) nounwind {1044; CHECK-LABEL: fmls_2d:1045; CHECK: // %bb.0:1046; CHECK-NEXT: ldr q1, [x0]1047; CHECK-NEXT: ldr q2, [x1]1048; CHECK-NEXT: ldr q0, [x2]1049; CHECK-NEXT: fmls v0.2d, v1.2d, v2.2d1050; CHECK-NEXT: ret1051 %tmp1 = load <2 x double>, ptr %A1052 %tmp2 = load <2 x double>, ptr %B1053 %tmp3 = load <2 x double>, ptr %C1054 %tmp4 = fsub <2 x double> <double -0.0, double -0.0>, %tmp21055 %tmp5 = call <2 x double> @llvm.fma.v2f64(<2 x double> %tmp1, <2 x double> %tmp4, <2 x double> %tmp3)1056 ret <2 x double> %tmp51057}1058 1059define <2 x float> @fmls_commuted_neg_2s(ptr %A, ptr %B, ptr %C) nounwind {1060; CHECK-LABEL: fmls_commuted_neg_2s:1061; CHECK: // %bb.0:1062; CHECK-NEXT: ldr d1, [x0]1063; CHECK-NEXT: ldr d2, [x1]1064; CHECK-NEXT: ldr d0, [x2]1065; CHECK-NEXT: fmls v0.2s, v1.2s, v2.2s1066; CHECK-NEXT: ret1067 %tmp1 = load <2 x float>, ptr %A1068 %tmp2 = load <2 x float>, ptr %B1069 %tmp3 = load <2 x float>, ptr %C1070 %tmp4 = fsub <2 x float> <float -0.0, float -0.0>, %tmp21071 %tmp5 = call <2 x float> @llvm.fma.v2f32(<2 x float> %tmp4, <2 x float> %tmp1, <2 x float> %tmp3)1072 ret <2 x float> %tmp51073}1074 1075define <4 x float> @fmls_commuted_neg_4s(ptr %A, ptr %B, ptr %C) nounwind {1076; CHECK-LABEL: fmls_commuted_neg_4s:1077; CHECK: // %bb.0:1078; CHECK-NEXT: ldr q1, [x0]1079; CHECK-NEXT: ldr q2, [x1]1080; CHECK-NEXT: ldr q0, [x2]1081; CHECK-NEXT: fmls v0.4s, v1.4s, v2.4s1082; CHECK-NEXT: ret1083 %tmp1 = load <4 x float>, ptr %A1084 %tmp2 = load <4 x float>, ptr %B1085 %tmp3 = load <4 x float>, ptr %C1086 %tmp4 = fsub <4 x float> <float -0.0, float -0.0, float -0.0, float -0.0>, %tmp21087 %tmp5 = call <4 x float> @llvm.fma.v4f32(<4 x float> %tmp4, <4 x float> %tmp1, <4 x float> %tmp3)1088 ret <4 x float> %tmp51089}1090 1091define <2 x double> @fmls_commuted_neg_2d(ptr %A, ptr %B, ptr %C) nounwind {1092; CHECK-LABEL: fmls_commuted_neg_2d:1093; CHECK: // %bb.0:1094; CHECK-NEXT: ldr q1, [x0]1095; CHECK-NEXT: ldr q2, [x1]1096; CHECK-NEXT: ldr q0, [x2]1097; CHECK-NEXT: fmls v0.2d, v1.2d, v2.2d1098; CHECK-NEXT: ret1099 %tmp1 = load <2 x double>, ptr %A1100 %tmp2 = load <2 x double>, ptr %B1101 %tmp3 = load <2 x double>, ptr %C1102 %tmp4 = fsub <2 x double> <double -0.0, double -0.0>, %tmp21103 %tmp5 = call <2 x double> @llvm.fma.v2f64(<2 x double> %tmp4, <2 x double> %tmp1, <2 x double> %tmp3)1104 ret <2 x double> %tmp51105}1106 1107define <2 x float> @fmls_indexed_2s(<2 x float> %a, <2 x float> %b, <2 x float> %c) nounwind readnone ssp {1108; CHECK-LABEL: fmls_indexed_2s:1109; CHECK: // %bb.0: // %entry1110; CHECK-NEXT: // kill: def $d1 killed $d1 def $q11111; CHECK-NEXT: fmls v0.2s, v2.2s, v1.s[0]1112; CHECK-NEXT: ret1113entry:1114 %0 = fsub <2 x float> <float -0.000000e+00, float -0.000000e+00>, %c1115 %lane = shufflevector <2 x float> %b, <2 x float> undef, <2 x i32> zeroinitializer1116 %fmls1 = tail call <2 x float> @llvm.fma.v2f32(<2 x float> %0, <2 x float> %lane, <2 x float> %a)1117 ret <2 x float> %fmls11118}1119 1120define <4 x float> @fmls_indexed_4s(<4 x float> %a, <4 x float> %b, <4 x float> %c) nounwind readnone ssp {1121; CHECK-LABEL: fmls_indexed_4s:1122; CHECK: // %bb.0: // %entry1123; CHECK-NEXT: fmls v0.4s, v2.4s, v1.s[0]1124; CHECK-NEXT: ret1125entry:1126 %0 = fsub <4 x float> <float -0.000000e+00, float -0.000000e+00, float -0.000000e+00, float -0.000000e+00>, %c1127 %lane = shufflevector <4 x float> %b, <4 x float> undef, <4 x i32> zeroinitializer1128 %fmls1 = tail call <4 x float> @llvm.fma.v4f32(<4 x float> %0, <4 x float> %lane, <4 x float> %a)1129 ret <4 x float> %fmls11130}1131 1132define <2 x double> @fmls_indexed_2d(<2 x double> %a, <2 x double> %b, <2 x double> %c) nounwind readnone ssp {1133; CHECK-LABEL: fmls_indexed_2d:1134; CHECK: // %bb.0: // %entry1135; CHECK-NEXT: fmls v0.2d, v2.2d, v1.d[0]1136; CHECK-NEXT: ret1137entry:1138 %0 = fsub <2 x double> <double -0.000000e+00, double -0.000000e+00>, %c1139 %lane = shufflevector <2 x double> %b, <2 x double> undef, <2 x i32> zeroinitializer1140 %fmls1 = tail call <2 x double> @llvm.fma.v2f64(<2 x double> %0, <2 x double> %lane, <2 x double> %a)1141 ret <2 x double> %fmls11142}1143 1144define <2 x float> @fmla_indexed_scalar_2s(<2 x float> %a, <2 x float> %b, float %c) nounwind readnone ssp {1145; CHECK-LABEL: fmla_indexed_scalar_2s:1146; CHECK: // %bb.0: // %entry1147; CHECK-NEXT: // kill: def $s2 killed $s2 def $d21148; CHECK-NEXT: fmla v0.2s, v1.2s, v2.2s1149; CHECK-NEXT: ret1150entry:1151 %v1 = insertelement <2 x float> undef, float %c, i32 01152 %v2 = insertelement <2 x float> %v1, float %c, i32 11153 %fmla1 = tail call <2 x float> @llvm.fma.v2f32(<2 x float> %v1, <2 x float> %b, <2 x float> %a) nounwind1154 ret <2 x float> %fmla11155}1156 1157define <4 x float> @fmla_indexed_scalar_4s(<4 x float> %a, <4 x float> %b, float %c) nounwind readnone ssp {1158; CHECK-LABEL: fmla_indexed_scalar_4s:1159; CHECK: // %bb.0: // %entry1160; CHECK-NEXT: // kill: def $s2 killed $s2 def $q21161; CHECK-NEXT: fmla v0.4s, v1.4s, v2.s[0]1162; CHECK-NEXT: ret1163entry:1164 %v1 = insertelement <4 x float> undef, float %c, i32 01165 %v2 = insertelement <4 x float> %v1, float %c, i32 11166 %v3 = insertelement <4 x float> %v2, float %c, i32 21167 %v4 = insertelement <4 x float> %v3, float %c, i32 31168 %fmla1 = tail call <4 x float> @llvm.fma.v4f32(<4 x float> %v4, <4 x float> %b, <4 x float> %a) nounwind1169 ret <4 x float> %fmla11170}1171 1172define <2 x double> @fmla_indexed_scalar_2d(<2 x double> %a, <2 x double> %b, double %c) nounwind readnone ssp {1173; CHECK-LABEL: fmla_indexed_scalar_2d:1174; CHECK: // %bb.0: // %entry1175; CHECK-NEXT: // kill: def $d2 killed $d2 def $q21176; CHECK-NEXT: fmla v0.2d, v1.2d, v2.d[0]1177; CHECK-NEXT: ret1178entry:1179 %v1 = insertelement <2 x double> undef, double %c, i32 01180 %v2 = insertelement <2 x double> %v1, double %c, i32 11181 %fmla1 = tail call <2 x double> @llvm.fma.v2f64(<2 x double> %v2, <2 x double> %b, <2 x double> %a) nounwind1182 ret <2 x double> %fmla11183}1184 1185define <2 x float> @fmls_indexed_2s_strict(<2 x float> %a, <2 x float> %b, <2 x float> %c) nounwind readnone ssp strictfp {1186; CHECK-LABEL: fmls_indexed_2s_strict:1187; CHECK: // %bb.0: // %entry1188; CHECK-NEXT: // kill: def $d1 killed $d1 def $q11189; CHECK-NEXT: fmls v0.2s, v2.2s, v1.s[0]1190; CHECK-NEXT: ret1191entry:1192 %0 = fneg <2 x float> %c1193 %lane = shufflevector <2 x float> %b, <2 x float> undef, <2 x i32> zeroinitializer1194 %fmls1 = tail call <2 x float> @llvm.experimental.constrained.fma.v2f32(<2 x float> %0, <2 x float> %lane, <2 x float> %a, metadata !"round.tonearest", metadata !"fpexcept.strict") #01195 ret <2 x float> %fmls11196}1197 1198define <4 x float> @fmls_indexed_4s_strict(<4 x float> %a, <4 x float> %b, <4 x float> %c) nounwind readnone ssp strictfp {1199; CHECK-LABEL: fmls_indexed_4s_strict:1200; CHECK: // %bb.0: // %entry1201; CHECK-NEXT: fmls v0.4s, v2.4s, v1.s[0]1202; CHECK-NEXT: ret1203entry:1204 %0 = fneg <4 x float> %c1205 %lane = shufflevector <4 x float> %b, <4 x float> undef, <4 x i32> zeroinitializer1206 %fmls1 = tail call <4 x float> @llvm.experimental.constrained.fma.v4f32(<4 x float> %0, <4 x float> %lane, <4 x float> %a, metadata !"round.tonearest", metadata !"fpexcept.strict") #01207 ret <4 x float> %fmls11208}1209 1210define <2 x double> @fmls_indexed_2d_strict(<2 x double> %a, <2 x double> %b, <2 x double> %c) nounwind readnone ssp strictfp {1211; CHECK-LABEL: fmls_indexed_2d_strict:1212; CHECK: // %bb.0: // %entry1213; CHECK-NEXT: fmls v0.2d, v2.2d, v1.d[0]1214; CHECK-NEXT: ret1215entry:1216 %0 = fneg <2 x double> %c1217 %lane = shufflevector <2 x double> %b, <2 x double> undef, <2 x i32> zeroinitializer1218 %fmls1 = tail call <2 x double> @llvm.experimental.constrained.fma.v2f64(<2 x double> %0, <2 x double> %lane, <2 x double> %a, metadata !"round.tonearest", metadata !"fpexcept.strict") #01219 ret <2 x double> %fmls11220}1221 1222define <2 x float> @fmla_indexed_scalar_2s_strict(<2 x float> %a, <2 x float> %b, float %c) nounwind readnone ssp strictfp {1223; CHECK-LABEL: fmla_indexed_scalar_2s_strict:1224; CHECK: // %bb.0: // %entry1225; CHECK-NEXT: // kill: def $s2 killed $s2 def $q21226; CHECK-NEXT: fmla v0.2s, v1.2s, v2.s[0]1227; CHECK-NEXT: ret1228entry:1229 %v1 = insertelement <2 x float> undef, float %c, i32 01230 %v2 = insertelement <2 x float> %v1, float %c, i32 11231 %fmla1 = tail call <2 x float> @llvm.experimental.constrained.fma.v2f32(<2 x float> %v2, <2 x float> %b, <2 x float> %a, metadata !"round.tonearest", metadata !"fpexcept.strict") #01232 ret <2 x float> %fmla11233}1234 1235define <4 x float> @fmla_indexed_scalar_4s_strict(<4 x float> %a, <4 x float> %b, float %c) nounwind readnone ssp strictfp {1236; CHECK-LABEL: fmla_indexed_scalar_4s_strict:1237; CHECK: // %bb.0: // %entry1238; CHECK-NEXT: // kill: def $s2 killed $s2 def $q21239; CHECK-NEXT: fmla v0.4s, v1.4s, v2.s[0]1240; CHECK-NEXT: ret1241entry:1242 %v1 = insertelement <4 x float> undef, float %c, i32 01243 %v2 = insertelement <4 x float> %v1, float %c, i32 11244 %v3 = insertelement <4 x float> %v2, float %c, i32 21245 %v4 = insertelement <4 x float> %v3, float %c, i32 31246 %fmla1 = tail call <4 x float> @llvm.experimental.constrained.fma.v4f32(<4 x float> %v4, <4 x float> %b, <4 x float> %a, metadata !"round.tonearest", metadata !"fpexcept.strict") #01247 ret <4 x float> %fmla11248}1249 1250define <2 x double> @fmla_indexed_scalar_2d_strict(<2 x double> %a, <2 x double> %b, double %c) nounwind readnone ssp strictfp {1251; CHECK-LABEL: fmla_indexed_scalar_2d_strict:1252; CHECK: // %bb.0: // %entry1253; CHECK-NEXT: // kill: def $d2 killed $d2 def $q21254; CHECK-NEXT: fmla v0.2d, v1.2d, v2.d[0]1255; CHECK-NEXT: ret1256entry:1257 %v1 = insertelement <2 x double> undef, double %c, i32 01258 %v2 = insertelement <2 x double> %v1, double %c, i32 11259 %fmla1 = tail call <2 x double> @llvm.experimental.constrained.fma.v2f64(<2 x double> %v2, <2 x double> %b, <2 x double> %a, metadata !"round.tonearest", metadata !"fpexcept.strict") #01260 ret <2 x double> %fmla11261}1262 1263attributes #0 = { strictfp }1264 1265declare <2 x float> @llvm.experimental.constrained.fma.v2f32(<2 x float>, <2 x float>, <2 x float>, metadata, metadata)1266declare <4 x float> @llvm.experimental.constrained.fma.v4f32(<4 x float>, <4 x float>, <4 x float>, metadata, metadata)1267declare <2 x double> @llvm.experimental.constrained.fma.v2f64(<2 x double>, <2 x double>, <2 x double>, metadata, metadata)1268 1269define <4 x i16> @mul_4h(<4 x i16> %A, <4 x i16> %B) nounwind {1270; CHECK-LABEL: mul_4h:1271; CHECK: // %bb.0:1272; CHECK-NEXT: // kill: def $d1 killed $d1 def $q11273; CHECK-NEXT: mul v0.4h, v0.4h, v1.h[1]1274; CHECK-NEXT: ret1275 %tmp3 = shufflevector <4 x i16> %B, <4 x i16> poison, <4 x i32> <i32 1, i32 1, i32 1, i32 1>1276 %tmp4 = mul <4 x i16> %A, %tmp31277 ret <4 x i16> %tmp41278}1279 1280define <8 x i16> @mul_8h(<8 x i16> %A, <8 x i16> %B) nounwind {1281; CHECK-LABEL: mul_8h:1282; CHECK: // %bb.0:1283; CHECK-NEXT: mul v0.8h, v0.8h, v1.h[1]1284; CHECK-NEXT: ret1285 %tmp3 = shufflevector <8 x i16> %B, <8 x i16> poison, <8 x i32> <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>1286 %tmp4 = mul <8 x i16> %A, %tmp31287 ret <8 x i16> %tmp41288}1289 1290define <2 x i32> @mul_2s(<2 x i32> %A, <2 x i32> %B) nounwind {1291; CHECK-LABEL: mul_2s:1292; CHECK: // %bb.0:1293; CHECK-NEXT: // kill: def $d1 killed $d1 def $q11294; CHECK-NEXT: mul v0.2s, v0.2s, v1.s[1]1295; CHECK-NEXT: ret1296 %tmp3 = shufflevector <2 x i32> %B, <2 x i32> poison, <2 x i32> <i32 1, i32 1>1297 %tmp4 = mul <2 x i32> %A, %tmp31298 ret <2 x i32> %tmp41299}1300 1301define <4 x i32> @mul_4s(<4 x i32> %A, <4 x i32> %B) nounwind {1302; CHECK-LABEL: mul_4s:1303; CHECK: // %bb.0:1304; CHECK-NEXT: mul v0.4s, v0.4s, v1.s[1]1305; CHECK-NEXT: ret1306 %tmp3 = shufflevector <4 x i32> %B, <4 x i32> poison, <4 x i32> <i32 1, i32 1, i32 1, i32 1>1307 %tmp4 = mul <4 x i32> %A, %tmp31308 ret <4 x i32> %tmp41309}1310 1311define <2 x i64> @mul_2d(<2 x i64> %A, <2 x i64> %B) nounwind {1312; CHECK-SD-LABEL: mul_2d:1313; CHECK-SD: // %bb.0:1314; CHECK-SD-NEXT: fmov x10, d11315; CHECK-SD-NEXT: fmov x11, d01316; CHECK-SD-NEXT: mov x8, v1.d[1]1317; CHECK-SD-NEXT: mov x9, v0.d[1]1318; CHECK-SD-NEXT: mul x10, x11, x101319; CHECK-SD-NEXT: mul x8, x9, x81320; CHECK-SD-NEXT: fmov d0, x101321; CHECK-SD-NEXT: mov v0.d[1], x81322; CHECK-SD-NEXT: ret1323;1324; CHECK-GI-LABEL: mul_2d:1325; CHECK-GI: // %bb.0:1326; CHECK-GI-NEXT: fmov x10, d01327; CHECK-GI-NEXT: fmov x11, d11328; CHECK-GI-NEXT: mov x8, v0.d[1]1329; CHECK-GI-NEXT: mov x9, v1.d[1]1330; CHECK-GI-NEXT: mul x10, x10, x111331; CHECK-GI-NEXT: mul x8, x8, x91332; CHECK-GI-NEXT: fmov d0, x101333; CHECK-GI-NEXT: mov v0.d[1], x81334; CHECK-GI-NEXT: ret1335 %tmp1 = mul <2 x i64> %A, %B1336 ret <2 x i64> %tmp11337}1338 1339define <2 x float> @fmul_lane_2s(<2 x float> %A, <2 x float> %B) nounwind {1340; CHECK-LABEL: fmul_lane_2s:1341; CHECK: // %bb.0:1342; CHECK-NEXT: // kill: def $d1 killed $d1 def $q11343; CHECK-NEXT: fmul v0.2s, v0.2s, v1.s[1]1344; CHECK-NEXT: ret1345 %tmp3 = shufflevector <2 x float> %B, <2 x float> poison, <2 x i32> <i32 1, i32 1>1346 %tmp4 = fmul <2 x float> %A, %tmp31347 ret <2 x float> %tmp41348}1349 1350define <4 x float> @fmul_lane_4s(<4 x float> %A, <4 x float> %B) nounwind {1351; CHECK-LABEL: fmul_lane_4s:1352; CHECK: // %bb.0:1353; CHECK-NEXT: fmul v0.4s, v0.4s, v1.s[1]1354; CHECK-NEXT: ret1355 %tmp3 = shufflevector <4 x float> %B, <4 x float> poison, <4 x i32> <i32 1, i32 1, i32 1, i32 1>1356 %tmp4 = fmul <4 x float> %A, %tmp31357 ret <4 x float> %tmp41358}1359 1360define <2 x double> @fmul_lane_2d(<2 x double> %A, <2 x double> %B) nounwind {1361; CHECK-LABEL: fmul_lane_2d:1362; CHECK: // %bb.0:1363; CHECK-NEXT: fmul v0.2d, v0.2d, v1.d[1]1364; CHECK-NEXT: ret1365 %tmp3 = shufflevector <2 x double> %B, <2 x double> poison, <2 x i32> <i32 1, i32 1>1366 %tmp4 = fmul <2 x double> %A, %tmp31367 ret <2 x double> %tmp41368}1369 1370define float @fmul_lane_s(float %A, <4 x float> %vec) nounwind {1371; CHECK-LABEL: fmul_lane_s:1372; CHECK: // %bb.0:1373; CHECK-NEXT: fmul s0, s0, v1.s[3]1374; CHECK-NEXT: ret1375 %B = extractelement <4 x float> %vec, i32 31376 %res = fmul float %A, %B1377 ret float %res1378}1379 1380define double @fmul_lane_d(double %A, <2 x double> %vec) nounwind {1381; CHECK-LABEL: fmul_lane_d:1382; CHECK: // %bb.0:1383; CHECK-NEXT: fmul d0, d0, v1.d[1]1384; CHECK-NEXT: ret1385 %B = extractelement <2 x double> %vec, i32 11386 %res = fmul double %A, %B1387 ret double %res1388}1389 1390 1391 1392define <2 x float> @fmulx_lane_2s(<2 x float> %A, <2 x float> %B) nounwind {1393; CHECK-LABEL: fmulx_lane_2s:1394; CHECK: // %bb.0:1395; CHECK-NEXT: // kill: def $d1 killed $d1 def $q11396; CHECK-NEXT: fmulx v0.2s, v0.2s, v1.s[1]1397; CHECK-NEXT: ret1398 %tmp3 = shufflevector <2 x float> %B, <2 x float> poison, <2 x i32> <i32 1, i32 1>1399 %tmp4 = call <2 x float> @llvm.aarch64.neon.fmulx.v2f32(<2 x float> %A, <2 x float> %tmp3)1400 ret <2 x float> %tmp41401}1402 1403define <4 x float> @fmulx_lane_4s(<4 x float> %A, <4 x float> %B) nounwind {1404; CHECK-LABEL: fmulx_lane_4s:1405; CHECK: // %bb.0:1406; CHECK-NEXT: fmulx v0.4s, v0.4s, v1.s[1]1407; CHECK-NEXT: ret1408 %tmp3 = shufflevector <4 x float> %B, <4 x float> poison, <4 x i32> <i32 1, i32 1, i32 1, i32 1>1409 %tmp4 = call <4 x float> @llvm.aarch64.neon.fmulx.v4f32(<4 x float> %A, <4 x float> %tmp3)1410 ret <4 x float> %tmp41411}1412 1413define <2 x double> @fmulx_lane_2d(<2 x double> %A, <2 x double> %B) nounwind {1414; CHECK-LABEL: fmulx_lane_2d:1415; CHECK: // %bb.0:1416; CHECK-NEXT: fmulx v0.2d, v0.2d, v1.d[1]1417; CHECK-NEXT: ret1418 %tmp3 = shufflevector <2 x double> %B, <2 x double> poison, <2 x i32> <i32 1, i32 1>1419 %tmp4 = call <2 x double> @llvm.aarch64.neon.fmulx.v2f64(<2 x double> %A, <2 x double> %tmp3)1420 ret <2 x double> %tmp41421}1422 1423define <4 x i16> @sqdmulh_lane_4h(<4 x i16> %A, <4 x i16> %B) nounwind {1424; CHECK-LABEL: sqdmulh_lane_4h:1425; CHECK: // %bb.0:1426; CHECK-NEXT: // kill: def $d1 killed $d1 def $q11427; CHECK-NEXT: sqdmulh v0.4h, v0.4h, v1.h[1]1428; CHECK-NEXT: ret1429 %tmp3 = shufflevector <4 x i16> %B, <4 x i16> poison, <4 x i32> <i32 1, i32 1, i32 1, i32 1>1430 %tmp4 = call <4 x i16> @llvm.aarch64.neon.sqdmulh.v4i16(<4 x i16> %A, <4 x i16> %tmp3)1431 ret <4 x i16> %tmp41432}1433 1434define <8 x i16> @sqdmulh_lane_8h(<8 x i16> %A, <8 x i16> %B) nounwind {1435; CHECK-LABEL: sqdmulh_lane_8h:1436; CHECK: // %bb.0:1437; CHECK-NEXT: sqdmulh v0.8h, v0.8h, v1.h[1]1438; CHECK-NEXT: ret1439 %tmp3 = shufflevector <8 x i16> %B, <8 x i16> poison, <8 x i32> <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>1440 %tmp4 = call <8 x i16> @llvm.aarch64.neon.sqdmulh.v8i16(<8 x i16> %A, <8 x i16> %tmp3)1441 ret <8 x i16> %tmp41442}1443 1444define <2 x i32> @sqdmulh_lane_2s(<2 x i32> %A, <2 x i32> %B) nounwind {1445; CHECK-LABEL: sqdmulh_lane_2s:1446; CHECK: // %bb.0:1447; CHECK-NEXT: // kill: def $d1 killed $d1 def $q11448; CHECK-NEXT: sqdmulh v0.2s, v0.2s, v1.s[1]1449; CHECK-NEXT: ret1450 %tmp3 = shufflevector <2 x i32> %B, <2 x i32> poison, <2 x i32> <i32 1, i32 1>1451 %tmp4 = call <2 x i32> @llvm.aarch64.neon.sqdmulh.v2i32(<2 x i32> %A, <2 x i32> %tmp3)1452 ret <2 x i32> %tmp41453}1454 1455define <4 x i32> @sqdmulh_lane_4s(<4 x i32> %A, <4 x i32> %B) nounwind {1456; CHECK-LABEL: sqdmulh_lane_4s:1457; CHECK: // %bb.0:1458; CHECK-NEXT: sqdmulh v0.4s, v0.4s, v1.s[1]1459; CHECK-NEXT: ret1460 %tmp3 = shufflevector <4 x i32> %B, <4 x i32> poison, <4 x i32> <i32 1, i32 1, i32 1, i32 1>1461 %tmp4 = call <4 x i32> @llvm.aarch64.neon.sqdmulh.v4i32(<4 x i32> %A, <4 x i32> %tmp3)1462 ret <4 x i32> %tmp41463}1464 1465define i32 @sqdmulh_lane_1s(i32 %A, <4 x i32> %B) nounwind {1466; CHECK-LABEL: sqdmulh_lane_1s:1467; CHECK: // %bb.0:1468; CHECK-NEXT: fmov s1, w01469; CHECK-NEXT: sqdmulh s0, s1, v0.s[1]1470; CHECK-NEXT: fmov w0, s01471; CHECK-NEXT: ret1472 %tmp1 = extractelement <4 x i32> %B, i32 11473 %tmp2 = call i32 @llvm.aarch64.neon.sqdmulh.i32(i32 %A, i32 %tmp1)1474 ret i32 %tmp21475}1476 1477define <4 x i16> @sqrdmulh_lane_4h(<4 x i16> %A, <4 x i16> %B) nounwind {1478; CHECK-LABEL: sqrdmulh_lane_4h:1479; CHECK: // %bb.0:1480; CHECK-NEXT: // kill: def $d1 killed $d1 def $q11481; CHECK-NEXT: sqrdmulh v0.4h, v0.4h, v1.h[1]1482; CHECK-NEXT: ret1483 %tmp3 = shufflevector <4 x i16> %B, <4 x i16> poison, <4 x i32> <i32 1, i32 1, i32 1, i32 1>1484 %tmp4 = call <4 x i16> @llvm.aarch64.neon.sqrdmulh.v4i16(<4 x i16> %A, <4 x i16> %tmp3)1485 ret <4 x i16> %tmp41486}1487 1488define <8 x i16> @sqrdmulh_lane_8h(<8 x i16> %A, <8 x i16> %B) nounwind {1489; CHECK-LABEL: sqrdmulh_lane_8h:1490; CHECK: // %bb.0:1491; CHECK-NEXT: sqrdmulh v0.8h, v0.8h, v1.h[1]1492; CHECK-NEXT: ret1493 %tmp3 = shufflevector <8 x i16> %B, <8 x i16> poison, <8 x i32> <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>1494 %tmp4 = call <8 x i16> @llvm.aarch64.neon.sqrdmulh.v8i16(<8 x i16> %A, <8 x i16> %tmp3)1495 ret <8 x i16> %tmp41496}1497 1498define <2 x i32> @sqrdmulh_lane_2s(<2 x i32> %A, <2 x i32> %B) nounwind {1499; CHECK-LABEL: sqrdmulh_lane_2s:1500; CHECK: // %bb.0:1501; CHECK-NEXT: // kill: def $d1 killed $d1 def $q11502; CHECK-NEXT: sqrdmulh v0.2s, v0.2s, v1.s[1]1503; CHECK-NEXT: ret1504 %tmp3 = shufflevector <2 x i32> %B, <2 x i32> poison, <2 x i32> <i32 1, i32 1>1505 %tmp4 = call <2 x i32> @llvm.aarch64.neon.sqrdmulh.v2i32(<2 x i32> %A, <2 x i32> %tmp3)1506 ret <2 x i32> %tmp41507}1508 1509define <4 x i32> @sqrdmulh_lane_4s(<4 x i32> %A, <4 x i32> %B) nounwind {1510; CHECK-LABEL: sqrdmulh_lane_4s:1511; CHECK: // %bb.0:1512; CHECK-NEXT: sqrdmulh v0.4s, v0.4s, v1.s[1]1513; CHECK-NEXT: ret1514 %tmp3 = shufflevector <4 x i32> %B, <4 x i32> poison, <4 x i32> <i32 1, i32 1, i32 1, i32 1>1515 %tmp4 = call <4 x i32> @llvm.aarch64.neon.sqrdmulh.v4i32(<4 x i32> %A, <4 x i32> %tmp3)1516 ret <4 x i32> %tmp41517}1518 1519define i32 @sqrdmulh_lane_1s(i32 %A, <4 x i32> %B) nounwind {1520; CHECK-LABEL: sqrdmulh_lane_1s:1521; CHECK: // %bb.0:1522; CHECK-NEXT: fmov s1, w01523; CHECK-NEXT: sqrdmulh s0, s1, v0.s[1]1524; CHECK-NEXT: fmov w0, s01525; CHECK-NEXT: ret1526 %tmp1 = extractelement <4 x i32> %B, i32 11527 %tmp2 = call i32 @llvm.aarch64.neon.sqrdmulh.i32(i32 %A, i32 %tmp1)1528 ret i32 %tmp21529}1530 1531define <4 x i32> @sqdmull_lane_4s(<4 x i16> %A, <4 x i16> %B) nounwind {1532; CHECK-LABEL: sqdmull_lane_4s:1533; CHECK: // %bb.0:1534; CHECK-NEXT: // kill: def $d1 killed $d1 def $q11535; CHECK-NEXT: sqdmull v0.4s, v0.4h, v1.h[1]1536; CHECK-NEXT: ret1537 %tmp3 = shufflevector <4 x i16> %B, <4 x i16> poison, <4 x i32> <i32 1, i32 1, i32 1, i32 1>1538 %tmp4 = call <4 x i32> @llvm.aarch64.neon.sqdmull.v4i32(<4 x i16> %A, <4 x i16> %tmp3)1539 ret <4 x i32> %tmp41540}1541 1542define <2 x i64> @sqdmull_lane_2d(<2 x i32> %A, <2 x i32> %B) nounwind {1543; CHECK-LABEL: sqdmull_lane_2d:1544; CHECK: // %bb.0:1545; CHECK-NEXT: // kill: def $d1 killed $d1 def $q11546; CHECK-NEXT: sqdmull v0.2d, v0.2s, v1.s[1]1547; CHECK-NEXT: ret1548 %tmp3 = shufflevector <2 x i32> %B, <2 x i32> poison, <2 x i32> <i32 1, i32 1>1549 %tmp4 = call <2 x i64> @llvm.aarch64.neon.sqdmull.v2i64(<2 x i32> %A, <2 x i32> %tmp3)1550 ret <2 x i64> %tmp41551}1552 1553define <4 x i32> @sqdmull2_lane_4s(<8 x i16> %A, <8 x i16> %B) nounwind {1554; CHECK-SD-LABEL: sqdmull2_lane_4s:1555; CHECK-SD: // %bb.0:1556; CHECK-SD-NEXT: sqdmull2 v0.4s, v0.8h, v1.h[1]1557; CHECK-SD-NEXT: ret1558;1559; CHECK-GI-LABEL: sqdmull2_lane_4s:1560; CHECK-GI: // %bb.0:1561; CHECK-GI-NEXT: mov d0, v0.d[1]1562; CHECK-GI-NEXT: sqdmull v0.4s, v0.4h, v1.h[1]1563; CHECK-GI-NEXT: ret1564 %tmp1 = shufflevector <8 x i16> %A, <8 x i16> undef, <4 x i32> <i32 4, i32 5, i32 6, i32 7>1565 %tmp2 = shufflevector <8 x i16> %B, <8 x i16> undef, <4 x i32> <i32 1, i32 1, i32 1, i32 1>1566 %tmp4 = call <4 x i32> @llvm.aarch64.neon.sqdmull.v4i32(<4 x i16> %tmp1, <4 x i16> %tmp2)1567 ret <4 x i32> %tmp41568}1569 1570define <2 x i64> @sqdmull2_lane_2d(<4 x i32> %A, <4 x i32> %B) nounwind {1571; CHECK-SD-LABEL: sqdmull2_lane_2d:1572; CHECK-SD: // %bb.0:1573; CHECK-SD-NEXT: sqdmull2 v0.2d, v0.4s, v1.s[1]1574; CHECK-SD-NEXT: ret1575;1576; CHECK-GI-LABEL: sqdmull2_lane_2d:1577; CHECK-GI: // %bb.0:1578; CHECK-GI-NEXT: mov d0, v0.d[1]1579; CHECK-GI-NEXT: sqdmull v0.2d, v0.2s, v1.s[1]1580; CHECK-GI-NEXT: ret1581 %tmp1 = shufflevector <4 x i32> %A, <4 x i32> undef, <2 x i32> <i32 2, i32 3>1582 %tmp2 = shufflevector <4 x i32> %B, <4 x i32> undef, <2 x i32> <i32 1, i32 1>1583 %tmp4 = call <2 x i64> @llvm.aarch64.neon.sqdmull.v2i64(<2 x i32> %tmp1, <2 x i32> %tmp2)1584 ret <2 x i64> %tmp41585}1586 1587define <4 x i32> @umull_lane_4s(<4 x i16> %A, <4 x i16> %B) nounwind {1588; CHECK-LABEL: umull_lane_4s:1589; CHECK: // %bb.0:1590; CHECK-NEXT: // kill: def $d1 killed $d1 def $q11591; CHECK-NEXT: umull v0.4s, v0.4h, v1.h[1]1592; CHECK-NEXT: ret1593 %tmp3 = shufflevector <4 x i16> %B, <4 x i16> poison, <4 x i32> <i32 1, i32 1, i32 1, i32 1>1594 %tmp4 = call <4 x i32> @llvm.aarch64.neon.umull.v4i32(<4 x i16> %A, <4 x i16> %tmp3)1595 ret <4 x i32> %tmp41596}1597 1598define <2 x i64> @umull_lane_2d(<2 x i32> %A, <2 x i32> %B) nounwind {1599; CHECK-LABEL: umull_lane_2d:1600; CHECK: // %bb.0:1601; CHECK-NEXT: // kill: def $d1 killed $d1 def $q11602; CHECK-NEXT: umull v0.2d, v0.2s, v1.s[1]1603; CHECK-NEXT: ret1604 %tmp3 = shufflevector <2 x i32> %B, <2 x i32> poison, <2 x i32> <i32 1, i32 1>1605 %tmp4 = call <2 x i64> @llvm.aarch64.neon.umull.v2i64(<2 x i32> %A, <2 x i32> %tmp3)1606 ret <2 x i64> %tmp41607}1608 1609define <4 x i32> @smull_lane_4s(<4 x i16> %A, <4 x i16> %B) nounwind {1610; CHECK-LABEL: smull_lane_4s:1611; CHECK: // %bb.0:1612; CHECK-NEXT: // kill: def $d1 killed $d1 def $q11613; CHECK-NEXT: smull v0.4s, v0.4h, v1.h[1]1614; CHECK-NEXT: ret1615 %tmp3 = shufflevector <4 x i16> %B, <4 x i16> poison, <4 x i32> <i32 1, i32 1, i32 1, i32 1>1616 %tmp4 = call <4 x i32> @llvm.aarch64.neon.smull.v4i32(<4 x i16> %A, <4 x i16> %tmp3)1617 ret <4 x i32> %tmp41618}1619 1620define <2 x i64> @smull_lane_2d(<2 x i32> %A, <2 x i32> %B) nounwind {1621; CHECK-LABEL: smull_lane_2d:1622; CHECK: // %bb.0:1623; CHECK-NEXT: // kill: def $d1 killed $d1 def $q11624; CHECK-NEXT: smull v0.2d, v0.2s, v1.s[1]1625; CHECK-NEXT: ret1626 %tmp3 = shufflevector <2 x i32> %B, <2 x i32> poison, <2 x i32> <i32 1, i32 1>1627 %tmp4 = call <2 x i64> @llvm.aarch64.neon.smull.v2i64(<2 x i32> %A, <2 x i32> %tmp3)1628 ret <2 x i64> %tmp41629}1630 1631define <4 x i32> @smlal_lane_4s(<4 x i16> %A, <4 x i16> %B, <4 x i32> %C) nounwind {1632; CHECK-LABEL: smlal_lane_4s:1633; CHECK: // %bb.0:1634; CHECK-NEXT: // kill: def $d1 killed $d1 def $q11635; CHECK-NEXT: smlal v2.4s, v0.4h, v1.h[1]1636; CHECK-NEXT: mov v0.16b, v2.16b1637; CHECK-NEXT: ret1638 %tmp4 = shufflevector <4 x i16> %B, <4 x i16> poison, <4 x i32> <i32 1, i32 1, i32 1, i32 1>1639 %tmp5 = call <4 x i32> @llvm.aarch64.neon.smull.v4i32(<4 x i16> %A, <4 x i16> %tmp4)1640 %tmp6 = add <4 x i32> %C, %tmp51641 ret <4 x i32> %tmp61642}1643 1644define <2 x i64> @smlal_lane_2d(<2 x i32> %A, <2 x i32> %B, <2 x i64> %C) nounwind {1645; CHECK-LABEL: smlal_lane_2d:1646; CHECK: // %bb.0:1647; CHECK-NEXT: // kill: def $d1 killed $d1 def $q11648; CHECK-NEXT: smlal v2.2d, v0.2s, v1.s[1]1649; CHECK-NEXT: mov v0.16b, v2.16b1650; CHECK-NEXT: ret1651 %tmp4 = shufflevector <2 x i32> %B, <2 x i32> poison, <2 x i32> <i32 1, i32 1>1652 %tmp5 = call <2 x i64> @llvm.aarch64.neon.smull.v2i64(<2 x i32> %A, <2 x i32> %tmp4)1653 %tmp6 = add <2 x i64> %C, %tmp51654 ret <2 x i64> %tmp61655}1656 1657define <4 x i32> @sqdmlal_lane_4s(<4 x i16> %A, <4 x i16> %B, <4 x i32> %C) nounwind {1658; CHECK-LABEL: sqdmlal_lane_4s:1659; CHECK: // %bb.0:1660; CHECK-NEXT: // kill: def $d1 killed $d1 def $q11661; CHECK-NEXT: sqdmlal v2.4s, v0.4h, v1.h[1]1662; CHECK-NEXT: mov v0.16b, v2.16b1663; CHECK-NEXT: ret1664 %tmp4 = shufflevector <4 x i16> %B, <4 x i16> poison, <4 x i32> <i32 1, i32 1, i32 1, i32 1>1665 %tmp5 = call <4 x i32> @llvm.aarch64.neon.sqdmull.v4i32(<4 x i16> %A, <4 x i16> %tmp4)1666 %tmp6 = call <4 x i32> @llvm.aarch64.neon.sqadd.v4i32(<4 x i32> %C, <4 x i32> %tmp5)1667 ret <4 x i32> %tmp61668}1669 1670define <2 x i64> @sqdmlal_lane_2d(<2 x i32> %A, <2 x i32> %B, <2 x i64> %C) nounwind {1671; CHECK-LABEL: sqdmlal_lane_2d:1672; CHECK: // %bb.0:1673; CHECK-NEXT: // kill: def $d1 killed $d1 def $q11674; CHECK-NEXT: sqdmlal v2.2d, v0.2s, v1.s[1]1675; CHECK-NEXT: mov v0.16b, v2.16b1676; CHECK-NEXT: ret1677 %tmp4 = shufflevector <2 x i32> %B, <2 x i32> poison, <2 x i32> <i32 1, i32 1>1678 %tmp5 = call <2 x i64> @llvm.aarch64.neon.sqdmull.v2i64(<2 x i32> %A, <2 x i32> %tmp4)1679 %tmp6 = call <2 x i64> @llvm.aarch64.neon.sqadd.v2i64(<2 x i64> %C, <2 x i64> %tmp5)1680 ret <2 x i64> %tmp61681}1682 1683define <4 x i32> @sqdmlal2_lane_4s(<8 x i16> %A, <8 x i16> %B, <4 x i32> %C) nounwind {1684; CHECK-SD-LABEL: sqdmlal2_lane_4s:1685; CHECK-SD: // %bb.0:1686; CHECK-SD-NEXT: sqdmlal2 v2.4s, v0.8h, v1.h[1]1687; CHECK-SD-NEXT: mov v0.16b, v2.16b1688; CHECK-SD-NEXT: ret1689;1690; CHECK-GI-LABEL: sqdmlal2_lane_4s:1691; CHECK-GI: // %bb.0:1692; CHECK-GI-NEXT: mov d3, v0.d[1]1693; CHECK-GI-NEXT: mov v0.16b, v2.16b1694; CHECK-GI-NEXT: sqdmlal v0.4s, v3.4h, v1.h[1]1695; CHECK-GI-NEXT: ret1696 %tmp1 = shufflevector <8 x i16> %A, <8 x i16> undef, <4 x i32> <i32 4, i32 5, i32 6, i32 7>1697 %tmp2 = shufflevector <8 x i16> %B, <8 x i16> undef, <4 x i32> <i32 1, i32 1, i32 1, i32 1>1698 %tmp5 = call <4 x i32> @llvm.aarch64.neon.sqdmull.v4i32(<4 x i16> %tmp1, <4 x i16> %tmp2)1699 %tmp6 = call <4 x i32> @llvm.aarch64.neon.sqadd.v4i32(<4 x i32> %C, <4 x i32> %tmp5)1700 ret <4 x i32> %tmp61701}1702 1703define <2 x i64> @sqdmlal2_lane_2d(<4 x i32> %A, <4 x i32> %B, <2 x i64> %C) nounwind {1704; CHECK-SD-LABEL: sqdmlal2_lane_2d:1705; CHECK-SD: // %bb.0:1706; CHECK-SD-NEXT: sqdmlal2 v2.2d, v0.4s, v1.s[1]1707; CHECK-SD-NEXT: mov v0.16b, v2.16b1708; CHECK-SD-NEXT: ret1709;1710; CHECK-GI-LABEL: sqdmlal2_lane_2d:1711; CHECK-GI: // %bb.0:1712; CHECK-GI-NEXT: mov d3, v0.d[1]1713; CHECK-GI-NEXT: mov v0.16b, v2.16b1714; CHECK-GI-NEXT: sqdmlal v0.2d, v3.2s, v1.s[1]1715; CHECK-GI-NEXT: ret1716 %tmp1 = shufflevector <4 x i32> %A, <4 x i32> undef, <2 x i32> <i32 2, i32 3>1717 %tmp2 = shufflevector <4 x i32> %B, <4 x i32> undef, <2 x i32> <i32 1, i32 1>1718 %tmp5 = call <2 x i64> @llvm.aarch64.neon.sqdmull.v2i64(<2 x i32> %tmp1, <2 x i32> %tmp2)1719 %tmp6 = call <2 x i64> @llvm.aarch64.neon.sqadd.v2i64(<2 x i64> %C, <2 x i64> %tmp5)1720 ret <2 x i64> %tmp61721}1722 1723define i32 @sqdmlal_lane_1s(i32 %A, i16 %B, <4 x i16> %C) nounwind {1724; CHECK-LABEL: sqdmlal_lane_1s:1725; CHECK: // %bb.0:1726; CHECK-NEXT: fmov s1, w11727; CHECK-NEXT: fmov s2, w01728; CHECK-NEXT: // kill: def $d0 killed $d0 def $q01729; CHECK-NEXT: sqdmlal s2, h1, v0.h[1]1730; CHECK-NEXT: fmov w0, s21731; CHECK-NEXT: ret1732 %lhs = insertelement <4 x i16> undef, i16 %B, i32 01733 %rhs = shufflevector <4 x i16> %C, <4 x i16> undef, <4 x i32> <i32 1, i32 undef, i32 undef, i32 undef>1734 %prod.vec = call <4 x i32> @llvm.aarch64.neon.sqdmull.v4i32(<4 x i16> %lhs, <4 x i16> %rhs)1735 %prod = extractelement <4 x i32> %prod.vec, i32 01736 %res = call i32 @llvm.aarch64.neon.sqadd.i32(i32 %A, i32 %prod)1737 ret i32 %res1738}1739declare i32 @llvm.aarch64.neon.sqadd.i32(i32, i32)1740 1741define i32 @sqdmlsl_lane_1s(i32 %A, i16 %B, <4 x i16> %C) nounwind {1742; CHECK-LABEL: sqdmlsl_lane_1s:1743; CHECK: // %bb.0:1744; CHECK-NEXT: fmov s1, w11745; CHECK-NEXT: fmov s2, w01746; CHECK-NEXT: // kill: def $d0 killed $d0 def $q01747; CHECK-NEXT: sqdmlsl s2, h1, v0.h[1]1748; CHECK-NEXT: fmov w0, s21749; CHECK-NEXT: ret1750 %lhs = insertelement <4 x i16> undef, i16 %B, i32 01751 %rhs = shufflevector <4 x i16> %C, <4 x i16> undef, <4 x i32> <i32 1, i32 undef, i32 undef, i32 undef>1752 %prod.vec = call <4 x i32> @llvm.aarch64.neon.sqdmull.v4i32(<4 x i16> %lhs, <4 x i16> %rhs)1753 %prod = extractelement <4 x i32> %prod.vec, i32 01754 %res = call i32 @llvm.aarch64.neon.sqsub.i32(i32 %A, i32 %prod)1755 ret i32 %res1756}1757declare i32 @llvm.aarch64.neon.sqsub.i32(i32, i32)1758 1759define i32 @sqadd_lane1_sqdmull4s(i32 %A, <4 x i16> %B, <4 x i16> %C) nounwind {1760; CHECK-SD-LABEL: sqadd_lane1_sqdmull4s:1761; CHECK-SD: // %bb.0:1762; CHECK-SD-NEXT: sqdmull v0.4s, v0.4h, v1.4h1763; CHECK-SD-NEXT: mov w8, v0.s[1]1764; CHECK-SD-NEXT: fmov s0, w01765; CHECK-SD-NEXT: fmov s1, w81766; CHECK-SD-NEXT: sqadd s0, s0, s11767; CHECK-SD-NEXT: fmov w0, s01768; CHECK-SD-NEXT: ret1769;1770; CHECK-GI-LABEL: sqadd_lane1_sqdmull4s:1771; CHECK-GI: // %bb.0:1772; CHECK-GI-NEXT: sqdmull v0.4s, v0.4h, v1.4h1773; CHECK-GI-NEXT: fmov s1, w01774; CHECK-GI-NEXT: mov s0, v0.s[1]1775; CHECK-GI-NEXT: sqadd s0, s1, s01776; CHECK-GI-NEXT: fmov w0, s01777; CHECK-GI-NEXT: ret1778 %prod.vec = call <4 x i32> @llvm.aarch64.neon.sqdmull.v4i32(<4 x i16> %B, <4 x i16> %C)1779 %prod = extractelement <4 x i32> %prod.vec, i32 11780 %res = call i32 @llvm.aarch64.neon.sqadd.i32(i32 %A, i32 %prod)1781 ret i32 %res1782}1783 1784define i32 @sqsub_lane1_sqdmull4s(i32 %A, <4 x i16> %B, <4 x i16> %C) nounwind {1785; CHECK-SD-LABEL: sqsub_lane1_sqdmull4s:1786; CHECK-SD: // %bb.0:1787; CHECK-SD-NEXT: sqdmull v0.4s, v0.4h, v1.4h1788; CHECK-SD-NEXT: mov w8, v0.s[1]1789; CHECK-SD-NEXT: fmov s0, w01790; CHECK-SD-NEXT: fmov s1, w81791; CHECK-SD-NEXT: sqsub s0, s0, s11792; CHECK-SD-NEXT: fmov w0, s01793; CHECK-SD-NEXT: ret1794;1795; CHECK-GI-LABEL: sqsub_lane1_sqdmull4s:1796; CHECK-GI: // %bb.0:1797; CHECK-GI-NEXT: sqdmull v0.4s, v0.4h, v1.4h1798; CHECK-GI-NEXT: fmov s1, w01799; CHECK-GI-NEXT: mov s0, v0.s[1]1800; CHECK-GI-NEXT: sqsub s0, s1, s01801; CHECK-GI-NEXT: fmov w0, s01802; CHECK-GI-NEXT: ret1803 %prod.vec = call <4 x i32> @llvm.aarch64.neon.sqdmull.v4i32(<4 x i16> %B, <4 x i16> %C)1804 %prod = extractelement <4 x i32> %prod.vec, i32 11805 %res = call i32 @llvm.aarch64.neon.sqsub.i32(i32 %A, i32 %prod)1806 ret i32 %res1807}1808 1809define i64 @sqdmlal_lane_1d(i64 %A, i32 %B, <2 x i32> %C) nounwind {1810; CHECK-LABEL: sqdmlal_lane_1d:1811; CHECK: // %bb.0:1812; CHECK-NEXT: fmov d1, x01813; CHECK-NEXT: fmov s2, w11814; CHECK-NEXT: // kill: def $d0 killed $d0 def $q01815; CHECK-NEXT: sqdmlal d1, s2, v0.s[1]1816; CHECK-NEXT: fmov x0, d11817; CHECK-NEXT: ret1818 %rhs = extractelement <2 x i32> %C, i32 11819 %prod = call i64 @llvm.aarch64.neon.sqdmulls.scalar(i32 %B, i32 %rhs)1820 %res = call i64 @llvm.aarch64.neon.sqadd.i64(i64 %A, i64 %prod)1821 ret i64 %res1822}1823declare i64 @llvm.aarch64.neon.sqdmulls.scalar(i32, i32)1824declare i64 @llvm.aarch64.neon.sqadd.i64(i64, i64)1825 1826define i64 @sqdmlsl_lane_1d(i64 %A, i32 %B, <2 x i32> %C) nounwind {1827; CHECK-LABEL: sqdmlsl_lane_1d:1828; CHECK: // %bb.0:1829; CHECK-NEXT: fmov d1, x01830; CHECK-NEXT: fmov s2, w11831; CHECK-NEXT: // kill: def $d0 killed $d0 def $q01832; CHECK-NEXT: sqdmlsl d1, s2, v0.s[1]1833; CHECK-NEXT: fmov x0, d11834; CHECK-NEXT: ret1835 %rhs = extractelement <2 x i32> %C, i32 11836 %prod = call i64 @llvm.aarch64.neon.sqdmulls.scalar(i32 %B, i32 %rhs)1837 %res = call i64 @llvm.aarch64.neon.sqsub.i64(i64 %A, i64 %prod)1838 ret i64 %res1839}1840declare i64 @llvm.aarch64.neon.sqsub.i64(i64, i64)1841 1842 1843define <4 x i32> @umlal_lane_4s(<4 x i16> %A, <4 x i16> %B, <4 x i32> %C) nounwind {1844; CHECK-LABEL: umlal_lane_4s:1845; CHECK: // %bb.0:1846; CHECK-NEXT: // kill: def $d1 killed $d1 def $q11847; CHECK-NEXT: umlal v2.4s, v0.4h, v1.h[1]1848; CHECK-NEXT: mov v0.16b, v2.16b1849; CHECK-NEXT: ret1850 %tmp4 = shufflevector <4 x i16> %B, <4 x i16> poison, <4 x i32> <i32 1, i32 1, i32 1, i32 1>1851 %tmp5 = call <4 x i32> @llvm.aarch64.neon.umull.v4i32(<4 x i16> %A, <4 x i16> %tmp4)1852 %tmp6 = add <4 x i32> %C, %tmp51853 ret <4 x i32> %tmp61854}1855 1856define <2 x i64> @umlal_lane_2d(<2 x i32> %A, <2 x i32> %B, <2 x i64> %C) nounwind {1857; CHECK-LABEL: umlal_lane_2d:1858; CHECK: // %bb.0:1859; CHECK-NEXT: // kill: def $d1 killed $d1 def $q11860; CHECK-NEXT: umlal v2.2d, v0.2s, v1.s[1]1861; CHECK-NEXT: mov v0.16b, v2.16b1862; CHECK-NEXT: ret1863 %tmp4 = shufflevector <2 x i32> %B, <2 x i32> poison, <2 x i32> <i32 1, i32 1>1864 %tmp5 = call <2 x i64> @llvm.aarch64.neon.umull.v2i64(<2 x i32> %A, <2 x i32> %tmp4)1865 %tmp6 = add <2 x i64> %C, %tmp51866 ret <2 x i64> %tmp61867}1868 1869 1870define <4 x i32> @smlsl_lane_4s(<4 x i16> %A, <4 x i16> %B, <4 x i32> %C) nounwind {1871; CHECK-LABEL: smlsl_lane_4s:1872; CHECK: // %bb.0:1873; CHECK-NEXT: // kill: def $d1 killed $d1 def $q11874; CHECK-NEXT: smlsl v2.4s, v0.4h, v1.h[1]1875; CHECK-NEXT: mov v0.16b, v2.16b1876; CHECK-NEXT: ret1877 %tmp4 = shufflevector <4 x i16> %B, <4 x i16> poison, <4 x i32> <i32 1, i32 1, i32 1, i32 1>1878 %tmp5 = call <4 x i32> @llvm.aarch64.neon.smull.v4i32(<4 x i16> %A, <4 x i16> %tmp4)1879 %tmp6 = sub <4 x i32> %C, %tmp51880 ret <4 x i32> %tmp61881}1882 1883define <2 x i64> @smlsl_lane_2d(<2 x i32> %A, <2 x i32> %B, <2 x i64> %C) nounwind {1884; CHECK-LABEL: smlsl_lane_2d:1885; CHECK: // %bb.0:1886; CHECK-NEXT: // kill: def $d1 killed $d1 def $q11887; CHECK-NEXT: smlsl v2.2d, v0.2s, v1.s[1]1888; CHECK-NEXT: mov v0.16b, v2.16b1889; CHECK-NEXT: ret1890 %tmp4 = shufflevector <2 x i32> %B, <2 x i32> poison, <2 x i32> <i32 1, i32 1>1891 %tmp5 = call <2 x i64> @llvm.aarch64.neon.smull.v2i64(<2 x i32> %A, <2 x i32> %tmp4)1892 %tmp6 = sub <2 x i64> %C, %tmp51893 ret <2 x i64> %tmp61894}1895 1896define <4 x i32> @sqdmlsl_lane_4s(<4 x i16> %A, <4 x i16> %B, <4 x i32> %C) nounwind {1897; CHECK-LABEL: sqdmlsl_lane_4s:1898; CHECK: // %bb.0:1899; CHECK-NEXT: // kill: def $d1 killed $d1 def $q11900; CHECK-NEXT: sqdmlsl v2.4s, v0.4h, v1.h[1]1901; CHECK-NEXT: mov v0.16b, v2.16b1902; CHECK-NEXT: ret1903 %tmp4 = shufflevector <4 x i16> %B, <4 x i16> poison, <4 x i32> <i32 1, i32 1, i32 1, i32 1>1904 %tmp5 = call <4 x i32> @llvm.aarch64.neon.sqdmull.v4i32(<4 x i16> %A, <4 x i16> %tmp4)1905 %tmp6 = call <4 x i32> @llvm.aarch64.neon.sqsub.v4i32(<4 x i32> %C, <4 x i32> %tmp5)1906 ret <4 x i32> %tmp61907}1908 1909define <2 x i64> @sqdmlsl_lane_2d(<2 x i32> %A, <2 x i32> %B, <2 x i64> %C) nounwind {1910; CHECK-LABEL: sqdmlsl_lane_2d:1911; CHECK: // %bb.0:1912; CHECK-NEXT: // kill: def $d1 killed $d1 def $q11913; CHECK-NEXT: sqdmlsl v2.2d, v0.2s, v1.s[1]1914; CHECK-NEXT: mov v0.16b, v2.16b1915; CHECK-NEXT: ret1916 %tmp4 = shufflevector <2 x i32> %B, <2 x i32> poison, <2 x i32> <i32 1, i32 1>1917 %tmp5 = call <2 x i64> @llvm.aarch64.neon.sqdmull.v2i64(<2 x i32> %A, <2 x i32> %tmp4)1918 %tmp6 = call <2 x i64> @llvm.aarch64.neon.sqsub.v2i64(<2 x i64> %C, <2 x i64> %tmp5)1919 ret <2 x i64> %tmp61920}1921 1922define <4 x i32> @sqdmlsl2_lane_4s(<8 x i16> %A, <8 x i16> %B, <4 x i32> %C) nounwind {1923; CHECK-SD-LABEL: sqdmlsl2_lane_4s:1924; CHECK-SD: // %bb.0:1925; CHECK-SD-NEXT: sqdmlsl2 v2.4s, v0.8h, v1.h[1]1926; CHECK-SD-NEXT: mov v0.16b, v2.16b1927; CHECK-SD-NEXT: ret1928;1929; CHECK-GI-LABEL: sqdmlsl2_lane_4s:1930; CHECK-GI: // %bb.0:1931; CHECK-GI-NEXT: mov d3, v0.d[1]1932; CHECK-GI-NEXT: mov v0.16b, v2.16b1933; CHECK-GI-NEXT: sqdmlsl v0.4s, v3.4h, v1.h[1]1934; CHECK-GI-NEXT: ret1935 %tmp1 = shufflevector <8 x i16> %A, <8 x i16> undef, <4 x i32> <i32 4, i32 5, i32 6, i32 7>1936 %tmp2 = shufflevector <8 x i16> %B, <8 x i16> undef, <4 x i32> <i32 1, i32 1, i32 1, i32 1>1937 %tmp5 = call <4 x i32> @llvm.aarch64.neon.sqdmull.v4i32(<4 x i16> %tmp1, <4 x i16> %tmp2)1938 %tmp6 = call <4 x i32> @llvm.aarch64.neon.sqsub.v4i32(<4 x i32> %C, <4 x i32> %tmp5)1939 ret <4 x i32> %tmp61940}1941 1942define <2 x i64> @sqdmlsl2_lane_2d(<4 x i32> %A, <4 x i32> %B, <2 x i64> %C) nounwind {1943; CHECK-SD-LABEL: sqdmlsl2_lane_2d:1944; CHECK-SD: // %bb.0:1945; CHECK-SD-NEXT: sqdmlsl2 v2.2d, v0.4s, v1.s[1]1946; CHECK-SD-NEXT: mov v0.16b, v2.16b1947; CHECK-SD-NEXT: ret1948;1949; CHECK-GI-LABEL: sqdmlsl2_lane_2d:1950; CHECK-GI: // %bb.0:1951; CHECK-GI-NEXT: mov d3, v0.d[1]1952; CHECK-GI-NEXT: mov v0.16b, v2.16b1953; CHECK-GI-NEXT: sqdmlsl v0.2d, v3.2s, v1.s[1]1954; CHECK-GI-NEXT: ret1955 %tmp1 = shufflevector <4 x i32> %A, <4 x i32> undef, <2 x i32> <i32 2, i32 3>1956 %tmp2 = shufflevector <4 x i32> %B, <4 x i32> undef, <2 x i32> <i32 1, i32 1>1957 %tmp5 = call <2 x i64> @llvm.aarch64.neon.sqdmull.v2i64(<2 x i32> %tmp1, <2 x i32> %tmp2)1958 %tmp6 = call <2 x i64> @llvm.aarch64.neon.sqsub.v2i64(<2 x i64> %C, <2 x i64> %tmp5)1959 ret <2 x i64> %tmp61960}1961 1962define <4 x i32> @umlsl_lane_4s(<4 x i16> %A, <4 x i16> %B, <4 x i32> %C) nounwind {1963; CHECK-LABEL: umlsl_lane_4s:1964; CHECK: // %bb.0:1965; CHECK-NEXT: // kill: def $d1 killed $d1 def $q11966; CHECK-NEXT: umlsl v2.4s, v0.4h, v1.h[1]1967; CHECK-NEXT: mov v0.16b, v2.16b1968; CHECK-NEXT: ret1969 %tmp4 = shufflevector <4 x i16> %B, <4 x i16> poison, <4 x i32> <i32 1, i32 1, i32 1, i32 1>1970 %tmp5 = call <4 x i32> @llvm.aarch64.neon.umull.v4i32(<4 x i16> %A, <4 x i16> %tmp4)1971 %tmp6 = sub <4 x i32> %C, %tmp51972 ret <4 x i32> %tmp61973}1974 1975define <2 x i64> @umlsl_lane_2d(<2 x i32> %A, <2 x i32> %B, <2 x i64> %C) nounwind {1976; CHECK-LABEL: umlsl_lane_2d:1977; CHECK: // %bb.0:1978; CHECK-NEXT: // kill: def $d1 killed $d1 def $q11979; CHECK-NEXT: umlsl v2.2d, v0.2s, v1.s[1]1980; CHECK-NEXT: mov v0.16b, v2.16b1981; CHECK-NEXT: ret1982 %tmp4 = shufflevector <2 x i32> %B, <2 x i32> poison, <2 x i32> <i32 1, i32 1>1983 %tmp5 = call <2 x i64> @llvm.aarch64.neon.umull.v2i64(<2 x i32> %A, <2 x i32> %tmp4)1984 %tmp6 = sub <2 x i64> %C, %tmp51985 ret <2 x i64> %tmp61986}1987 1988; Scalar FMULX1989define float @fmulxs(float %a, float %b) nounwind {1990; CHECK-LABEL: fmulxs:1991; CHECK: // %bb.0:1992; CHECK-NEXT: fmulx s0, s0, s11993; CHECK-NEXT: ret1994 %fmulx.i = tail call float @llvm.aarch64.neon.fmulx.f32(float %a, float %b) nounwind1995 ret float %fmulx.i1996}1997 1998define double @fmulxd(double %a, double %b) nounwind {1999; CHECK-LABEL: fmulxd:2000; CHECK: // %bb.0:2001; CHECK-NEXT: fmulx d0, d0, d12002; CHECK-NEXT: ret2003 %fmulx.i = tail call double @llvm.aarch64.neon.fmulx.f64(double %a, double %b) nounwind2004 ret double %fmulx.i2005}2006 2007define float @fmulxs_lane(float %a, <4 x float> %vec) nounwind {2008; CHECK-LABEL: fmulxs_lane:2009; CHECK: // %bb.0:2010; CHECK-NEXT: fmulx s0, s0, v1.s[3]2011; CHECK-NEXT: ret2012 %b = extractelement <4 x float> %vec, i32 32013 %fmulx.i = tail call float @llvm.aarch64.neon.fmulx.f32(float %a, float %b) nounwind2014 ret float %fmulx.i2015}2016 2017define double @fmulxd_lane(double %a, <2 x double> %vec) nounwind {2018; CHECK-LABEL: fmulxd_lane:2019; CHECK: // %bb.0:2020; CHECK-NEXT: fmulx d0, d0, v1.d[1]2021; CHECK-NEXT: ret2022 %b = extractelement <2 x double> %vec, i32 12023 %fmulx.i = tail call double @llvm.aarch64.neon.fmulx.f64(double %a, double %b) nounwind2024 ret double %fmulx.i2025}2026 2027declare double @llvm.aarch64.neon.fmulx.f64(double, double) nounwind readnone2028declare float @llvm.aarch64.neon.fmulx.f32(float, float) nounwind readnone2029 2030 2031define <8 x i16> @smull2_8h_simple(<16 x i8> %a, <16 x i8> %b) nounwind {2032; CHECK-LABEL: smull2_8h_simple:2033; CHECK: // %bb.0:2034; CHECK-NEXT: smull2 v0.8h, v0.16b, v1.16b2035; CHECK-NEXT: ret2036 %1 = shufflevector <16 x i8> %a, <16 x i8> undef, <8 x i32> <i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>2037 %2 = shufflevector <16 x i8> %b, <16 x i8> undef, <8 x i32> <i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>2038 %3 = tail call <8 x i16> @llvm.aarch64.neon.smull.v8i16(<8 x i8> %1, <8 x i8> %2) #22039 ret <8 x i16> %32040}2041 2042define <8 x i16> @foo0(<16 x i8> %a, <16 x i8> %b) nounwind {2043; CHECK-LABEL: foo0:2044; CHECK: // %bb.0:2045; CHECK-NEXT: smull2 v0.8h, v0.16b, v1.16b2046; CHECK-NEXT: ret2047 %tmp = bitcast <16 x i8> %a to <2 x i64>2048 %shuffle.i.i = shufflevector <2 x i64> %tmp, <2 x i64> undef, <1 x i32> <i32 1>2049 %tmp1 = bitcast <1 x i64> %shuffle.i.i to <8 x i8>2050 %tmp2 = bitcast <16 x i8> %b to <2 x i64>2051 %shuffle.i3.i = shufflevector <2 x i64> %tmp2, <2 x i64> undef, <1 x i32> <i32 1>2052 %tmp3 = bitcast <1 x i64> %shuffle.i3.i to <8 x i8>2053 %vmull.i.i = tail call <8 x i16> @llvm.aarch64.neon.smull.v8i16(<8 x i8> %tmp1, <8 x i8> %tmp3) nounwind2054 ret <8 x i16> %vmull.i.i2055}2056 2057define <4 x i32> @foo1(<8 x i16> %a, <8 x i16> %b) nounwind {2058; CHECK-LABEL: foo1:2059; CHECK: // %bb.0:2060; CHECK-NEXT: smull2 v0.4s, v0.8h, v1.8h2061; CHECK-NEXT: ret2062 %tmp = bitcast <8 x i16> %a to <2 x i64>2063 %shuffle.i.i = shufflevector <2 x i64> %tmp, <2 x i64> undef, <1 x i32> <i32 1>2064 %tmp1 = bitcast <1 x i64> %shuffle.i.i to <4 x i16>2065 %tmp2 = bitcast <8 x i16> %b to <2 x i64>2066 %shuffle.i3.i = shufflevector <2 x i64> %tmp2, <2 x i64> undef, <1 x i32> <i32 1>2067 %tmp3 = bitcast <1 x i64> %shuffle.i3.i to <4 x i16>2068 %vmull2.i.i = tail call <4 x i32> @llvm.aarch64.neon.smull.v4i32(<4 x i16> %tmp1, <4 x i16> %tmp3) nounwind2069 ret <4 x i32> %vmull2.i.i2070}2071 2072define <2 x i64> @foo2(<4 x i32> %a, <4 x i32> %b) nounwind {2073; CHECK-LABEL: foo2:2074; CHECK: // %bb.0:2075; CHECK-NEXT: smull2 v0.2d, v0.4s, v1.4s2076; CHECK-NEXT: ret2077 %tmp = bitcast <4 x i32> %a to <2 x i64>2078 %shuffle.i.i = shufflevector <2 x i64> %tmp, <2 x i64> undef, <1 x i32> <i32 1>2079 %tmp1 = bitcast <1 x i64> %shuffle.i.i to <2 x i32>2080 %tmp2 = bitcast <4 x i32> %b to <2 x i64>2081 %shuffle.i3.i = shufflevector <2 x i64> %tmp2, <2 x i64> undef, <1 x i32> <i32 1>2082 %tmp3 = bitcast <1 x i64> %shuffle.i3.i to <2 x i32>2083 %vmull2.i.i = tail call <2 x i64> @llvm.aarch64.neon.smull.v2i64(<2 x i32> %tmp1, <2 x i32> %tmp3) nounwind2084 ret <2 x i64> %vmull2.i.i2085}2086 2087define <8 x i16> @foo3(<16 x i8> %a, <16 x i8> %b) nounwind {2088; CHECK-LABEL: foo3:2089; CHECK: // %bb.0:2090; CHECK-NEXT: umull2 v0.8h, v0.16b, v1.16b2091; CHECK-NEXT: ret2092 %tmp = bitcast <16 x i8> %a to <2 x i64>2093 %shuffle.i.i = shufflevector <2 x i64> %tmp, <2 x i64> undef, <1 x i32> <i32 1>2094 %tmp1 = bitcast <1 x i64> %shuffle.i.i to <8 x i8>2095 %tmp2 = bitcast <16 x i8> %b to <2 x i64>2096 %shuffle.i3.i = shufflevector <2 x i64> %tmp2, <2 x i64> undef, <1 x i32> <i32 1>2097 %tmp3 = bitcast <1 x i64> %shuffle.i3.i to <8 x i8>2098 %vmull.i.i = tail call <8 x i16> @llvm.aarch64.neon.umull.v8i16(<8 x i8> %tmp1, <8 x i8> %tmp3) nounwind2099 ret <8 x i16> %vmull.i.i2100}2101 2102define <4 x i32> @foo4(<8 x i16> %a, <8 x i16> %b) nounwind {2103; CHECK-LABEL: foo4:2104; CHECK: // %bb.0:2105; CHECK-NEXT: umull2 v0.4s, v0.8h, v1.8h2106; CHECK-NEXT: ret2107 %tmp = bitcast <8 x i16> %a to <2 x i64>2108 %shuffle.i.i = shufflevector <2 x i64> %tmp, <2 x i64> undef, <1 x i32> <i32 1>2109 %tmp1 = bitcast <1 x i64> %shuffle.i.i to <4 x i16>2110 %tmp2 = bitcast <8 x i16> %b to <2 x i64>2111 %shuffle.i3.i = shufflevector <2 x i64> %tmp2, <2 x i64> undef, <1 x i32> <i32 1>2112 %tmp3 = bitcast <1 x i64> %shuffle.i3.i to <4 x i16>2113 %vmull2.i.i = tail call <4 x i32> @llvm.aarch64.neon.umull.v4i32(<4 x i16> %tmp1, <4 x i16> %tmp3) nounwind2114 ret <4 x i32> %vmull2.i.i2115}2116 2117define <2 x i64> @foo5(<4 x i32> %a, <4 x i32> %b) nounwind {2118; CHECK-LABEL: foo5:2119; CHECK: // %bb.0:2120; CHECK-NEXT: umull2 v0.2d, v0.4s, v1.4s2121; CHECK-NEXT: ret2122 %tmp = bitcast <4 x i32> %a to <2 x i64>2123 %shuffle.i.i = shufflevector <2 x i64> %tmp, <2 x i64> undef, <1 x i32> <i32 1>2124 %tmp1 = bitcast <1 x i64> %shuffle.i.i to <2 x i32>2125 %tmp2 = bitcast <4 x i32> %b to <2 x i64>2126 %shuffle.i3.i = shufflevector <2 x i64> %tmp2, <2 x i64> undef, <1 x i32> <i32 1>2127 %tmp3 = bitcast <1 x i64> %shuffle.i3.i to <2 x i32>2128 %vmull2.i.i = tail call <2 x i64> @llvm.aarch64.neon.umull.v2i64(<2 x i32> %tmp1, <2 x i32> %tmp3) nounwind2129 ret <2 x i64> %vmull2.i.i2130}2131 2132define <4 x i32> @foo6(<4 x i32> %a, <8 x i16> %b, <4 x i16> %c) nounwind readnone optsize ssp {2133; CHECK-SD-LABEL: foo6:2134; CHECK-SD: // %bb.0: // %entry2135; CHECK-SD-NEXT: // kill: def $d2 killed $d2 def $q22136; CHECK-SD-NEXT: smull2 v0.4s, v1.8h, v2.h[1]2137; CHECK-SD-NEXT: ret2138;2139; CHECK-GI-LABEL: foo6:2140; CHECK-GI: // %bb.0: // %entry2141; CHECK-GI-NEXT: mov d0, v1.d[1]2142; CHECK-GI-NEXT: // kill: def $d2 killed $d2 def $q22143; CHECK-GI-NEXT: smull v0.4s, v0.4h, v2.h[1]2144; CHECK-GI-NEXT: ret2145entry:2146 %0 = bitcast <8 x i16> %b to <2 x i64>2147 %shuffle.i = shufflevector <2 x i64> %0, <2 x i64> undef, <1 x i32> <i32 1>2148 %1 = bitcast <1 x i64> %shuffle.i to <4 x i16>2149 %shuffle = shufflevector <4 x i16> %c, <4 x i16> undef, <4 x i32> <i32 1, i32 1, i32 1, i32 1>2150 %vmull2.i = tail call <4 x i32> @llvm.aarch64.neon.smull.v4i32(<4 x i16> %1, <4 x i16> %shuffle) nounwind2151 ret <4 x i32> %vmull2.i2152}2153 2154define <4 x i32> @foo6a(<4 x i32> %a, <8 x i16> %b, <4 x i16> %c) nounwind readnone optsize ssp {2155; CHECK-LABEL: foo6a:2156; CHECK: // %bb.0: // %entry2157; CHECK-NEXT: // kill: def $d2 killed $d2 def $q22158; CHECK-NEXT: smull v0.4s, v1.4h, v2.h[1]2159; CHECK-NEXT: ret2160entry:2161 %0 = bitcast <8 x i16> %b to <2 x i64>2162 %shuffle.i = shufflevector <2 x i64> %0, <2 x i64> undef, <1 x i32> <i32 0>2163 %1 = bitcast <1 x i64> %shuffle.i to <4 x i16>2164 %shuffle = shufflevector <4 x i16> %c, <4 x i16> undef, <4 x i32> <i32 1, i32 1, i32 1, i32 1>2165 %vmull2.i = tail call <4 x i32> @llvm.aarch64.neon.smull.v4i32(<4 x i16> %1, <4 x i16> %shuffle) nounwind2166 ret <4 x i32> %vmull2.i2167}2168 2169define <2 x i64> @foo7(<2 x i64> %a, <4 x i32> %b, <2 x i32> %c) nounwind readnone optsize ssp {2170; CHECK-SD-LABEL: foo7:2171; CHECK-SD: // %bb.0: // %entry2172; CHECK-SD-NEXT: // kill: def $d2 killed $d2 def $q22173; CHECK-SD-NEXT: smull2 v0.2d, v1.4s, v2.s[1]2174; CHECK-SD-NEXT: ret2175;2176; CHECK-GI-LABEL: foo7:2177; CHECK-GI: // %bb.0: // %entry2178; CHECK-GI-NEXT: mov d0, v1.d[1]2179; CHECK-GI-NEXT: // kill: def $d2 killed $d2 def $q22180; CHECK-GI-NEXT: smull v0.2d, v0.2s, v2.s[1]2181; CHECK-GI-NEXT: ret2182entry:2183 %0 = bitcast <4 x i32> %b to <2 x i64>2184 %shuffle.i = shufflevector <2 x i64> %0, <2 x i64> undef, <1 x i32> <i32 1>2185 %1 = bitcast <1 x i64> %shuffle.i to <2 x i32>2186 %shuffle = shufflevector <2 x i32> %c, <2 x i32> undef, <2 x i32> <i32 1, i32 1>2187 %vmull2.i = tail call <2 x i64> @llvm.aarch64.neon.smull.v2i64(<2 x i32> %1, <2 x i32> %shuffle) nounwind2188 ret <2 x i64> %vmull2.i2189}2190 2191define <2 x i64> @foo7a(<2 x i64> %a, <4 x i32> %b, <2 x i32> %c) nounwind readnone optsize ssp {2192; CHECK-LABEL: foo7a:2193; CHECK: // %bb.0: // %entry2194; CHECK-NEXT: // kill: def $d2 killed $d2 def $q22195; CHECK-NEXT: smull v0.2d, v1.2s, v2.s[1]2196; CHECK-NEXT: ret2197entry:2198 %0 = bitcast <4 x i32> %b to <2 x i64>2199 %shuffle.i = shufflevector <2 x i64> %0, <2 x i64> undef, <1 x i32> <i32 0>2200 %1 = bitcast <1 x i64> %shuffle.i to <2 x i32>2201 %shuffle = shufflevector <2 x i32> %c, <2 x i32> undef, <2 x i32> <i32 1, i32 1>2202 %vmull2.i = tail call <2 x i64> @llvm.aarch64.neon.smull.v2i64(<2 x i32> %1, <2 x i32> %shuffle) nounwind2203 ret <2 x i64> %vmull2.i2204}2205 2206 2207define <4 x i32> @foo8(<4 x i32> %a, <8 x i16> %b, <4 x i16> %c) nounwind readnone optsize ssp {2208; CHECK-SD-LABEL: foo8:2209; CHECK-SD: // %bb.0: // %entry2210; CHECK-SD-NEXT: // kill: def $d2 killed $d2 def $q22211; CHECK-SD-NEXT: umull2 v0.4s, v1.8h, v2.h[1]2212; CHECK-SD-NEXT: ret2213;2214; CHECK-GI-LABEL: foo8:2215; CHECK-GI: // %bb.0: // %entry2216; CHECK-GI-NEXT: mov d0, v1.d[1]2217; CHECK-GI-NEXT: // kill: def $d2 killed $d2 def $q22218; CHECK-GI-NEXT: umull v0.4s, v0.4h, v2.h[1]2219; CHECK-GI-NEXT: ret2220entry:2221 %0 = bitcast <8 x i16> %b to <2 x i64>2222 %shuffle.i = shufflevector <2 x i64> %0, <2 x i64> undef, <1 x i32> <i32 1>2223 %1 = bitcast <1 x i64> %shuffle.i to <4 x i16>2224 %shuffle = shufflevector <4 x i16> %c, <4 x i16> undef, <4 x i32> <i32 1, i32 1, i32 1, i32 1>2225 %vmull2.i = tail call <4 x i32> @llvm.aarch64.neon.umull.v4i32(<4 x i16> %1, <4 x i16> %shuffle) nounwind2226 ret <4 x i32> %vmull2.i2227}2228 2229define <4 x i32> @foo8a(<4 x i32> %a, <8 x i16> %b, <4 x i16> %c) nounwind readnone optsize ssp {2230; CHECK-LABEL: foo8a:2231; CHECK: // %bb.0: // %entry2232; CHECK-NEXT: // kill: def $d2 killed $d2 def $q22233; CHECK-NEXT: umull v0.4s, v1.4h, v2.h[1]2234; CHECK-NEXT: ret2235entry:2236 %0 = bitcast <8 x i16> %b to <2 x i64>2237 %shuffle.i = shufflevector <2 x i64> %0, <2 x i64> undef, <1 x i32> <i32 0>2238 %1 = bitcast <1 x i64> %shuffle.i to <4 x i16>2239 %shuffle = shufflevector <4 x i16> %c, <4 x i16> undef, <4 x i32> <i32 1, i32 1, i32 1, i32 1>2240 %vmull2.i = tail call <4 x i32> @llvm.aarch64.neon.umull.v4i32(<4 x i16> %1, <4 x i16> %shuffle) nounwind2241 ret <4 x i32> %vmull2.i2242}2243 2244define <2 x i64> @foo9(<2 x i64> %a, <4 x i32> %b, <2 x i32> %c) nounwind readnone optsize ssp {2245; CHECK-SD-LABEL: foo9:2246; CHECK-SD: // %bb.0: // %entry2247; CHECK-SD-NEXT: // kill: def $d2 killed $d2 def $q22248; CHECK-SD-NEXT: umull2 v0.2d, v1.4s, v2.s[1]2249; CHECK-SD-NEXT: ret2250;2251; CHECK-GI-LABEL: foo9:2252; CHECK-GI: // %bb.0: // %entry2253; CHECK-GI-NEXT: mov d0, v1.d[1]2254; CHECK-GI-NEXT: // kill: def $d2 killed $d2 def $q22255; CHECK-GI-NEXT: umull v0.2d, v0.2s, v2.s[1]2256; CHECK-GI-NEXT: ret2257entry:2258 %0 = bitcast <4 x i32> %b to <2 x i64>2259 %shuffle.i = shufflevector <2 x i64> %0, <2 x i64> undef, <1 x i32> <i32 1>2260 %1 = bitcast <1 x i64> %shuffle.i to <2 x i32>2261 %shuffle = shufflevector <2 x i32> %c, <2 x i32> undef, <2 x i32> <i32 1, i32 1>2262 %vmull2.i = tail call <2 x i64> @llvm.aarch64.neon.umull.v2i64(<2 x i32> %1, <2 x i32> %shuffle) nounwind2263 ret <2 x i64> %vmull2.i2264}2265 2266define <2 x i64> @foo9a(<2 x i64> %a, <4 x i32> %b, <2 x i32> %c) nounwind readnone optsize ssp {2267; CHECK-LABEL: foo9a:2268; CHECK: // %bb.0: // %entry2269; CHECK-NEXT: // kill: def $d2 killed $d2 def $q22270; CHECK-NEXT: umull v0.2d, v1.2s, v2.s[1]2271; CHECK-NEXT: ret2272entry:2273 %0 = bitcast <4 x i32> %b to <2 x i64>2274 %shuffle.i = shufflevector <2 x i64> %0, <2 x i64> undef, <1 x i32> <i32 0>2275 %1 = bitcast <1 x i64> %shuffle.i to <2 x i32>2276 %shuffle = shufflevector <2 x i32> %c, <2 x i32> undef, <2 x i32> <i32 1, i32 1>2277 %vmull2.i = tail call <2 x i64> @llvm.aarch64.neon.umull.v2i64(<2 x i32> %1, <2 x i32> %shuffle) nounwind2278 ret <2 x i64> %vmull2.i2279}2280 2281define <8 x i16> @bar0(<8 x i16> %a, <16 x i8> %b, <16 x i8> %c) nounwind {2282; CHECK-LABEL: bar0:2283; CHECK: // %bb.0:2284; CHECK-NEXT: smlal2 v0.8h, v1.16b, v2.16b2285; CHECK-NEXT: ret2286 %tmp = bitcast <16 x i8> %b to <2 x i64>2287 %shuffle.i.i.i = shufflevector <2 x i64> %tmp, <2 x i64> undef, <1 x i32> <i32 1>2288 %tmp1 = bitcast <1 x i64> %shuffle.i.i.i to <8 x i8>2289 %tmp2 = bitcast <16 x i8> %c to <2 x i64>2290 %shuffle.i3.i.i = shufflevector <2 x i64> %tmp2, <2 x i64> undef, <1 x i32> <i32 1>2291 %tmp3 = bitcast <1 x i64> %shuffle.i3.i.i to <8 x i8>2292 %vmull.i.i.i = tail call <8 x i16> @llvm.aarch64.neon.smull.v8i16(<8 x i8> %tmp1, <8 x i8> %tmp3) nounwind2293 %add.i = add <8 x i16> %vmull.i.i.i, %a2294 ret <8 x i16> %add.i2295}2296 2297define <4 x i32> @bar1(<4 x i32> %a, <8 x i16> %b, <8 x i16> %c) nounwind {2298; CHECK-LABEL: bar1:2299; CHECK: // %bb.0:2300; CHECK-NEXT: smlal2 v0.4s, v1.8h, v2.8h2301; CHECK-NEXT: ret2302 %tmp = bitcast <8 x i16> %b to <2 x i64>2303 %shuffle.i.i.i = shufflevector <2 x i64> %tmp, <2 x i64> undef, <1 x i32> <i32 1>2304 %tmp1 = bitcast <1 x i64> %shuffle.i.i.i to <4 x i16>2305 %tmp2 = bitcast <8 x i16> %c to <2 x i64>2306 %shuffle.i3.i.i = shufflevector <2 x i64> %tmp2, <2 x i64> undef, <1 x i32> <i32 1>2307 %tmp3 = bitcast <1 x i64> %shuffle.i3.i.i to <4 x i16>2308 %vmull2.i.i.i = tail call <4 x i32> @llvm.aarch64.neon.smull.v4i32(<4 x i16> %tmp1, <4 x i16> %tmp3) nounwind2309 %add.i = add <4 x i32> %vmull2.i.i.i, %a2310 ret <4 x i32> %add.i2311}2312 2313define <2 x i64> @bar2(<2 x i64> %a, <4 x i32> %b, <4 x i32> %c) nounwind {2314; CHECK-LABEL: bar2:2315; CHECK: // %bb.0:2316; CHECK-NEXT: smlal2 v0.2d, v1.4s, v2.4s2317; CHECK-NEXT: ret2318 %tmp = bitcast <4 x i32> %b to <2 x i64>2319 %shuffle.i.i.i = shufflevector <2 x i64> %tmp, <2 x i64> undef, <1 x i32> <i32 1>2320 %tmp1 = bitcast <1 x i64> %shuffle.i.i.i to <2 x i32>2321 %tmp2 = bitcast <4 x i32> %c to <2 x i64>2322 %shuffle.i3.i.i = shufflevector <2 x i64> %tmp2, <2 x i64> undef, <1 x i32> <i32 1>2323 %tmp3 = bitcast <1 x i64> %shuffle.i3.i.i to <2 x i32>2324 %vmull2.i.i.i = tail call <2 x i64> @llvm.aarch64.neon.smull.v2i64(<2 x i32> %tmp1, <2 x i32> %tmp3) nounwind2325 %add.i = add <2 x i64> %vmull2.i.i.i, %a2326 ret <2 x i64> %add.i2327}2328 2329define <8 x i16> @bar3(<8 x i16> %a, <16 x i8> %b, <16 x i8> %c) nounwind {2330; CHECK-LABEL: bar3:2331; CHECK: // %bb.0:2332; CHECK-NEXT: umlal2 v0.8h, v1.16b, v2.16b2333; CHECK-NEXT: ret2334 %tmp = bitcast <16 x i8> %b to <2 x i64>2335 %shuffle.i.i.i = shufflevector <2 x i64> %tmp, <2 x i64> undef, <1 x i32> <i32 1>2336 %tmp1 = bitcast <1 x i64> %shuffle.i.i.i to <8 x i8>2337 %tmp2 = bitcast <16 x i8> %c to <2 x i64>2338 %shuffle.i3.i.i = shufflevector <2 x i64> %tmp2, <2 x i64> undef, <1 x i32> <i32 1>2339 %tmp3 = bitcast <1 x i64> %shuffle.i3.i.i to <8 x i8>2340 %vmull.i.i.i = tail call <8 x i16> @llvm.aarch64.neon.umull.v8i16(<8 x i8> %tmp1, <8 x i8> %tmp3) nounwind2341 %add.i = add <8 x i16> %vmull.i.i.i, %a2342 ret <8 x i16> %add.i2343}2344 2345define <4 x i32> @bar4(<4 x i32> %a, <8 x i16> %b, <8 x i16> %c) nounwind {2346; CHECK-LABEL: bar4:2347; CHECK: // %bb.0:2348; CHECK-NEXT: umlal2 v0.4s, v1.8h, v2.8h2349; CHECK-NEXT: ret2350 %tmp = bitcast <8 x i16> %b to <2 x i64>2351 %shuffle.i.i.i = shufflevector <2 x i64> %tmp, <2 x i64> undef, <1 x i32> <i32 1>2352 %tmp1 = bitcast <1 x i64> %shuffle.i.i.i to <4 x i16>2353 %tmp2 = bitcast <8 x i16> %c to <2 x i64>2354 %shuffle.i3.i.i = shufflevector <2 x i64> %tmp2, <2 x i64> undef, <1 x i32> <i32 1>2355 %tmp3 = bitcast <1 x i64> %shuffle.i3.i.i to <4 x i16>2356 %vmull2.i.i.i = tail call <4 x i32> @llvm.aarch64.neon.umull.v4i32(<4 x i16> %tmp1, <4 x i16> %tmp3) nounwind2357 %add.i = add <4 x i32> %vmull2.i.i.i, %a2358 ret <4 x i32> %add.i2359}2360 2361define <2 x i64> @bar5(<2 x i64> %a, <4 x i32> %b, <4 x i32> %c) nounwind {2362; CHECK-LABEL: bar5:2363; CHECK: // %bb.0:2364; CHECK-NEXT: umlal2 v0.2d, v1.4s, v2.4s2365; CHECK-NEXT: ret2366 %tmp = bitcast <4 x i32> %b to <2 x i64>2367 %shuffle.i.i.i = shufflevector <2 x i64> %tmp, <2 x i64> undef, <1 x i32> <i32 1>2368 %tmp1 = bitcast <1 x i64> %shuffle.i.i.i to <2 x i32>2369 %tmp2 = bitcast <4 x i32> %c to <2 x i64>2370 %shuffle.i3.i.i = shufflevector <2 x i64> %tmp2, <2 x i64> undef, <1 x i32> <i32 1>2371 %tmp3 = bitcast <1 x i64> %shuffle.i3.i.i to <2 x i32>2372 %vmull2.i.i.i = tail call <2 x i64> @llvm.aarch64.neon.umull.v2i64(<2 x i32> %tmp1, <2 x i32> %tmp3) nounwind2373 %add.i = add <2 x i64> %vmull2.i.i.i, %a2374 ret <2 x i64> %add.i2375}2376 2377define <4 x i32> @mlal2_1(<4 x i32> %a, <8 x i16> %b, <4 x i16> %c) nounwind {2378; CHECK-SD-LABEL: mlal2_1:2379; CHECK-SD: // %bb.0:2380; CHECK-SD-NEXT: // kill: def $d2 killed $d2 def $q22381; CHECK-SD-NEXT: smlal2 v0.4s, v1.8h, v2.h[3]2382; CHECK-SD-NEXT: ret2383;2384; CHECK-GI-LABEL: mlal2_1:2385; CHECK-GI: // %bb.0:2386; CHECK-GI-NEXT: // kill: def $d2 killed $d2 def $q22387; CHECK-GI-NEXT: dup v2.8h, v2.h[3]2388; CHECK-GI-NEXT: smlal2 v0.4s, v1.8h, v2.8h2389; CHECK-GI-NEXT: ret2390 %shuffle = shufflevector <4 x i16> %c, <4 x i16> undef, <8 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>2391 %tmp = bitcast <8 x i16> %b to <2 x i64>2392 %shuffle.i.i = shufflevector <2 x i64> %tmp, <2 x i64> undef, <1 x i32> <i32 1>2393 %tmp1 = bitcast <1 x i64> %shuffle.i.i to <4 x i16>2394 %tmp2 = bitcast <8 x i16> %shuffle to <2 x i64>2395 %shuffle.i3.i = shufflevector <2 x i64> %tmp2, <2 x i64> undef, <1 x i32> <i32 1>2396 %tmp3 = bitcast <1 x i64> %shuffle.i3.i to <4 x i16>2397 %vmull2.i.i = tail call <4 x i32> @llvm.aarch64.neon.smull.v4i32(<4 x i16> %tmp1, <4 x i16> %tmp3) nounwind2398 %add = add <4 x i32> %vmull2.i.i, %a2399 ret <4 x i32> %add2400}2401 2402define <2 x i64> @mlal2_2(<2 x i64> %a, <4 x i32> %b, <2 x i32> %c) nounwind {2403; CHECK-SD-LABEL: mlal2_2:2404; CHECK-SD: // %bb.0:2405; CHECK-SD-NEXT: // kill: def $d2 killed $d2 def $q22406; CHECK-SD-NEXT: smlal2 v0.2d, v1.4s, v2.s[1]2407; CHECK-SD-NEXT: ret2408;2409; CHECK-GI-LABEL: mlal2_2:2410; CHECK-GI: // %bb.0:2411; CHECK-GI-NEXT: // kill: def $d2 killed $d2 def $q22412; CHECK-GI-NEXT: dup v2.4s, v2.s[1]2413; CHECK-GI-NEXT: smlal2 v0.2d, v1.4s, v2.4s2414; CHECK-GI-NEXT: ret2415 %shuffle = shufflevector <2 x i32> %c, <2 x i32> undef, <4 x i32> <i32 1, i32 1, i32 1, i32 1>2416 %tmp = bitcast <4 x i32> %b to <2 x i64>2417 %shuffle.i.i = shufflevector <2 x i64> %tmp, <2 x i64> undef, <1 x i32> <i32 1>2418 %tmp1 = bitcast <1 x i64> %shuffle.i.i to <2 x i32>2419 %tmp2 = bitcast <4 x i32> %shuffle to <2 x i64>2420 %shuffle.i3.i = shufflevector <2 x i64> %tmp2, <2 x i64> undef, <1 x i32> <i32 1>2421 %tmp3 = bitcast <1 x i64> %shuffle.i3.i to <2 x i32>2422 %vmull2.i.i = tail call <2 x i64> @llvm.aarch64.neon.smull.v2i64(<2 x i32> %tmp1, <2 x i32> %tmp3) nounwind2423 %add = add <2 x i64> %vmull2.i.i, %a2424 ret <2 x i64> %add2425}2426 2427define <4 x i32> @mlal2_4(<4 x i32> %a, <8 x i16> %b, <4 x i16> %c) nounwind {2428; CHECK-SD-LABEL: mlal2_4:2429; CHECK-SD: // %bb.0:2430; CHECK-SD-NEXT: // kill: def $d2 killed $d2 def $q22431; CHECK-SD-NEXT: umlal2 v0.4s, v1.8h, v2.h[2]2432; CHECK-SD-NEXT: ret2433;2434; CHECK-GI-LABEL: mlal2_4:2435; CHECK-GI: // %bb.0:2436; CHECK-GI-NEXT: // kill: def $d2 killed $d2 def $q22437; CHECK-GI-NEXT: dup v2.8h, v2.h[2]2438; CHECK-GI-NEXT: umlal2 v0.4s, v1.8h, v2.8h2439; CHECK-GI-NEXT: ret2440 %shuffle = shufflevector <4 x i16> %c, <4 x i16> undef, <8 x i32> <i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2>2441 %tmp = bitcast <8 x i16> %b to <2 x i64>2442 %shuffle.i.i = shufflevector <2 x i64> %tmp, <2 x i64> undef, <1 x i32> <i32 1>2443 %tmp1 = bitcast <1 x i64> %shuffle.i.i to <4 x i16>2444 %tmp2 = bitcast <8 x i16> %shuffle to <2 x i64>2445 %shuffle.i3.i = shufflevector <2 x i64> %tmp2, <2 x i64> undef, <1 x i32> <i32 1>2446 %tmp3 = bitcast <1 x i64> %shuffle.i3.i to <4 x i16>2447 %vmull2.i.i = tail call <4 x i32> @llvm.aarch64.neon.umull.v4i32(<4 x i16> %tmp1, <4 x i16> %tmp3) nounwind2448 %add = add <4 x i32> %vmull2.i.i, %a2449 ret <4 x i32> %add2450}2451 2452define <2 x i64> @mlal2_5(<2 x i64> %a, <4 x i32> %b, <2 x i32> %c) nounwind {2453; CHECK-SD-LABEL: mlal2_5:2454; CHECK-SD: // %bb.0:2455; CHECK-SD-NEXT: // kill: def $d2 killed $d2 def $q22456; CHECK-SD-NEXT: umlal2 v0.2d, v1.4s, v2.s[0]2457; CHECK-SD-NEXT: ret2458;2459; CHECK-GI-LABEL: mlal2_5:2460; CHECK-GI: // %bb.0:2461; CHECK-GI-NEXT: // kill: def $d2 killed $d2 def $q22462; CHECK-GI-NEXT: dup v2.4s, v2.s[0]2463; CHECK-GI-NEXT: umlal2 v0.2d, v1.4s, v2.4s2464; CHECK-GI-NEXT: ret2465 %shuffle = shufflevector <2 x i32> %c, <2 x i32> undef, <4 x i32> zeroinitializer2466 %tmp = bitcast <4 x i32> %b to <2 x i64>2467 %shuffle.i.i = shufflevector <2 x i64> %tmp, <2 x i64> undef, <1 x i32> <i32 1>2468 %tmp1 = bitcast <1 x i64> %shuffle.i.i to <2 x i32>2469 %tmp2 = bitcast <4 x i32> %shuffle to <2 x i64>2470 %shuffle.i3.i = shufflevector <2 x i64> %tmp2, <2 x i64> undef, <1 x i32> <i32 1>2471 %tmp3 = bitcast <1 x i64> %shuffle.i3.i to <2 x i32>2472 %vmull2.i.i = tail call <2 x i64> @llvm.aarch64.neon.umull.v2i64(<2 x i32> %tmp1, <2 x i32> %tmp3) nounwind2473 %add = add <2 x i64> %vmull2.i.i, %a2474 ret <2 x i64> %add2475}2476 2477; rdar://123285022478define <2 x double> @vmulq_n_f64(<2 x double> %x, double %y) nounwind readnone ssp {2479; CHECK-LABEL: vmulq_n_f64:2480; CHECK: // %bb.0: // %entry2481; CHECK-NEXT: // kill: def $d1 killed $d1 def $q12482; CHECK-NEXT: fmul v0.2d, v0.2d, v1.d[0]2483; CHECK-NEXT: ret2484entry:2485 %vecinit.i = insertelement <2 x double> undef, double %y, i32 02486 %vecinit1.i = insertelement <2 x double> %vecinit.i, double %y, i32 12487 %mul.i = fmul <2 x double> %vecinit1.i, %x2488 ret <2 x double> %mul.i2489}2490 2491define <4 x float> @vmulq_n_f32(<4 x float> %x, float %y) nounwind readnone ssp {2492; CHECK-LABEL: vmulq_n_f32:2493; CHECK: // %bb.0: // %entry2494; CHECK-NEXT: // kill: def $s1 killed $s1 def $q12495; CHECK-NEXT: fmul v0.4s, v0.4s, v1.s[0]2496; CHECK-NEXT: ret2497entry:2498 %vecinit.i = insertelement <4 x float> undef, float %y, i32 02499 %vecinit1.i = insertelement <4 x float> %vecinit.i, float %y, i32 12500 %vecinit2.i = insertelement <4 x float> %vecinit1.i, float %y, i32 22501 %vecinit3.i = insertelement <4 x float> %vecinit2.i, float %y, i32 32502 %mul.i = fmul <4 x float> %vecinit3.i, %x2503 ret <4 x float> %mul.i2504}2505 2506define <2 x float> @vmul_n_f32(<2 x float> %x, float %y) nounwind readnone ssp {2507; CHECK-LABEL: vmul_n_f32:2508; CHECK: // %bb.0: // %entry2509; CHECK-NEXT: // kill: def $s1 killed $s1 def $q12510; CHECK-NEXT: fmul v0.2s, v0.2s, v1.s[0]2511; CHECK-NEXT: ret2512entry:2513 %vecinit.i = insertelement <2 x float> undef, float %y, i32 02514 %vecinit1.i = insertelement <2 x float> %vecinit.i, float %y, i32 12515 %mul.i = fmul <2 x float> %vecinit1.i, %x2516 ret <2 x float> %mul.i2517}2518 2519define <4 x i16> @vmla_laneq_s16_test(<4 x i16> %a, <4 x i16> %b, <8 x i16> %c) nounwind readnone ssp {2520; CHECK-LABEL: vmla_laneq_s16_test:2521; CHECK: // %bb.0: // %entry2522; CHECK-NEXT: mla v0.4h, v1.4h, v2.h[6]2523; CHECK-NEXT: ret2524entry:2525 %shuffle = shufflevector <8 x i16> %c, <8 x i16> undef, <4 x i32> <i32 6, i32 6, i32 6, i32 6>2526 %mul = mul <4 x i16> %shuffle, %b2527 %add = add <4 x i16> %mul, %a2528 ret <4 x i16> %add2529}2530 2531define <2 x i32> @vmla_laneq_s32_test(<2 x i32> %a, <2 x i32> %b, <4 x i32> %c) nounwind readnone ssp {2532; CHECK-LABEL: vmla_laneq_s32_test:2533; CHECK: // %bb.0: // %entry2534; CHECK-NEXT: mla v0.2s, v1.2s, v2.s[3]2535; CHECK-NEXT: ret2536entry:2537 %shuffle = shufflevector <4 x i32> %c, <4 x i32> undef, <2 x i32> <i32 3, i32 3>2538 %mul = mul <2 x i32> %shuffle, %b2539 %add = add <2 x i32> %mul, %a2540 ret <2 x i32> %add2541}2542 2543define <8 x i16> @not_really_vmlaq_laneq_s16_test(<8 x i16> %a, <8 x i16> %b, <8 x i16> %c) nounwind readnone ssp {2544; CHECK-SD-LABEL: not_really_vmlaq_laneq_s16_test:2545; CHECK-SD: // %bb.0: // %entry2546; CHECK-SD-NEXT: mla v0.8h, v1.8h, v2.h[5]2547; CHECK-SD-NEXT: ret2548;2549; CHECK-GI-LABEL: not_really_vmlaq_laneq_s16_test:2550; CHECK-GI: // %bb.0: // %entry2551; CHECK-GI-NEXT: ext v2.16b, v2.16b, v0.16b, #82552; CHECK-GI-NEXT: mla v0.8h, v1.8h, v2.h[1]2553; CHECK-GI-NEXT: ret2554entry:2555 %shuffle1 = shufflevector <8 x i16> %c, <8 x i16> undef, <4 x i32> <i32 4, i32 5, i32 6, i32 7>2556 %shuffle2 = shufflevector <4 x i16> %shuffle1, <4 x i16> undef, <8 x i32> <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>2557 %mul = mul <8 x i16> %shuffle2, %b2558 %add = add <8 x i16> %mul, %a2559 ret <8 x i16> %add2560}2561 2562define <4 x i32> @not_really_vmlaq_laneq_s32_test(<4 x i32> %a, <4 x i32> %b, <4 x i32> %c) nounwind readnone ssp {2563; CHECK-SD-LABEL: not_really_vmlaq_laneq_s32_test:2564; CHECK-SD: // %bb.0: // %entry2565; CHECK-SD-NEXT: mla v0.4s, v1.4s, v2.s[3]2566; CHECK-SD-NEXT: ret2567;2568; CHECK-GI-LABEL: not_really_vmlaq_laneq_s32_test:2569; CHECK-GI: // %bb.0: // %entry2570; CHECK-GI-NEXT: ext v2.16b, v2.16b, v0.16b, #82571; CHECK-GI-NEXT: mla v0.4s, v1.4s, v2.s[1]2572; CHECK-GI-NEXT: ret2573entry:2574 %shuffle1 = shufflevector <4 x i32> %c, <4 x i32> undef, <2 x i32> <i32 2, i32 3>2575 %shuffle2 = shufflevector <2 x i32> %shuffle1, <2 x i32> undef, <4 x i32> <i32 1, i32 1, i32 1, i32 1>2576 %mul = mul <4 x i32> %shuffle2, %b2577 %add = add <4 x i32> %mul, %a2578 ret <4 x i32> %add2579}2580 2581define <4 x i32> @vmull_laneq_s16_test(<4 x i16> %a, <8 x i16> %b) nounwind readnone ssp {2582; CHECK-LABEL: vmull_laneq_s16_test:2583; CHECK: // %bb.0: // %entry2584; CHECK-NEXT: smull v0.4s, v0.4h, v1.h[6]2585; CHECK-NEXT: ret2586entry:2587 %shuffle = shufflevector <8 x i16> %b, <8 x i16> undef, <4 x i32> <i32 6, i32 6, i32 6, i32 6>2588 %vmull2.i = tail call <4 x i32> @llvm.aarch64.neon.smull.v4i32(<4 x i16> %a, <4 x i16> %shuffle) #22589 ret <4 x i32> %vmull2.i2590}2591 2592define <2 x i64> @vmull_laneq_s32_test(<2 x i32> %a, <4 x i32> %b) nounwind readnone ssp {2593; CHECK-LABEL: vmull_laneq_s32_test:2594; CHECK: // %bb.0: // %entry2595; CHECK-NEXT: smull v0.2d, v0.2s, v1.s[2]2596; CHECK-NEXT: ret2597entry:2598 %shuffle = shufflevector <4 x i32> %b, <4 x i32> undef, <2 x i32> <i32 2, i32 2>2599 %vmull2.i = tail call <2 x i64> @llvm.aarch64.neon.smull.v2i64(<2 x i32> %a, <2 x i32> %shuffle) #22600 ret <2 x i64> %vmull2.i2601}2602define <4 x i32> @vmull_laneq_u16_test(<4 x i16> %a, <8 x i16> %b) nounwind readnone ssp {2603; CHECK-LABEL: vmull_laneq_u16_test:2604; CHECK: // %bb.0: // %entry2605; CHECK-NEXT: umull v0.4s, v0.4h, v1.h[6]2606; CHECK-NEXT: ret2607entry:2608 %shuffle = shufflevector <8 x i16> %b, <8 x i16> undef, <4 x i32> <i32 6, i32 6, i32 6, i32 6>2609 %vmull2.i = tail call <4 x i32> @llvm.aarch64.neon.umull.v4i32(<4 x i16> %a, <4 x i16> %shuffle) #22610 ret <4 x i32> %vmull2.i2611}2612 2613define <2 x i64> @vmull_laneq_u32_test(<2 x i32> %a, <4 x i32> %b) nounwind readnone ssp {2614; CHECK-LABEL: vmull_laneq_u32_test:2615; CHECK: // %bb.0: // %entry2616; CHECK-NEXT: umull v0.2d, v0.2s, v1.s[2]2617; CHECK-NEXT: ret2618entry:2619 %shuffle = shufflevector <4 x i32> %b, <4 x i32> undef, <2 x i32> <i32 2, i32 2>2620 %vmull2.i = tail call <2 x i64> @llvm.aarch64.neon.umull.v2i64(<2 x i32> %a, <2 x i32> %shuffle) #22621 ret <2 x i64> %vmull2.i2622}2623 2624define <4 x i32> @vmull_low_n_s16_test(<4 x i32> %a, <8 x i16> %b, <4 x i16> %c, i32 %d) nounwind readnone optsize ssp {2625; CHECK-LABEL: vmull_low_n_s16_test:2626; CHECK: // %bb.0: // %entry2627; CHECK-NEXT: dup v0.4h, w02628; CHECK-NEXT: smull v0.4s, v1.4h, v0.4h2629; CHECK-NEXT: ret2630entry:2631 %conv = trunc i32 %d to i162632 %0 = bitcast <8 x i16> %b to <2 x i64>2633 %shuffle.i.i = shufflevector <2 x i64> %0, <2 x i64> undef, <1 x i32> <i32 0>2634 %1 = bitcast <1 x i64> %shuffle.i.i to <4 x i16>2635 %vecinit.i = insertelement <4 x i16> undef, i16 %conv, i32 02636 %vecinit1.i = insertelement <4 x i16> %vecinit.i, i16 %conv, i32 12637 %vecinit2.i = insertelement <4 x i16> %vecinit1.i, i16 %conv, i32 22638 %vecinit3.i = insertelement <4 x i16> %vecinit2.i, i16 %conv, i32 32639 %vmull2.i.i = tail call <4 x i32> @llvm.aarch64.neon.smull.v4i32(<4 x i16> %1, <4 x i16> %vecinit3.i) nounwind2640 ret <4 x i32> %vmull2.i.i2641}2642 2643define <4 x i32> @vmull_high_n_s16_test(<4 x i32> %a, <8 x i16> %b, <4 x i16> %c, i32 %d) nounwind readnone optsize ssp {2644; CHECK-SD-LABEL: vmull_high_n_s16_test:2645; CHECK-SD: // %bb.0: // %entry2646; CHECK-SD-NEXT: dup v0.8h, w02647; CHECK-SD-NEXT: smull2 v0.4s, v1.8h, v0.8h2648; CHECK-SD-NEXT: ret2649;2650; CHECK-GI-LABEL: vmull_high_n_s16_test:2651; CHECK-GI: // %bb.0: // %entry2652; CHECK-GI-NEXT: mov d0, v1.d[1]2653; CHECK-GI-NEXT: dup v1.4h, w02654; CHECK-GI-NEXT: smull v0.4s, v0.4h, v1.4h2655; CHECK-GI-NEXT: ret2656entry:2657 %conv = trunc i32 %d to i162658 %0 = bitcast <8 x i16> %b to <2 x i64>2659 %shuffle.i.i = shufflevector <2 x i64> %0, <2 x i64> undef, <1 x i32> <i32 1>2660 %1 = bitcast <1 x i64> %shuffle.i.i to <4 x i16>2661 %vecinit.i = insertelement <4 x i16> undef, i16 %conv, i32 02662 %vecinit1.i = insertelement <4 x i16> %vecinit.i, i16 %conv, i32 12663 %vecinit2.i = insertelement <4 x i16> %vecinit1.i, i16 %conv, i32 22664 %vecinit3.i = insertelement <4 x i16> %vecinit2.i, i16 %conv, i32 32665 %vmull2.i.i = tail call <4 x i32> @llvm.aarch64.neon.smull.v4i32(<4 x i16> %1, <4 x i16> %vecinit3.i) nounwind2666 ret <4 x i32> %vmull2.i.i2667}2668 2669define <2 x i64> @vmull_high_n_s32_test(<2 x i64> %a, <4 x i32> %b, <2 x i32> %c, i32 %d) nounwind readnone optsize ssp {2670; CHECK-SD-LABEL: vmull_high_n_s32_test:2671; CHECK-SD: // %bb.0: // %entry2672; CHECK-SD-NEXT: dup v0.4s, w02673; CHECK-SD-NEXT: smull2 v0.2d, v1.4s, v0.4s2674; CHECK-SD-NEXT: ret2675;2676; CHECK-GI-LABEL: vmull_high_n_s32_test:2677; CHECK-GI: // %bb.0: // %entry2678; CHECK-GI-NEXT: mov d0, v1.d[1]2679; CHECK-GI-NEXT: dup v1.2s, w02680; CHECK-GI-NEXT: smull v0.2d, v0.2s, v1.2s2681; CHECK-GI-NEXT: ret2682entry:2683 %0 = bitcast <4 x i32> %b to <2 x i64>2684 %shuffle.i.i = shufflevector <2 x i64> %0, <2 x i64> undef, <1 x i32> <i32 1>2685 %1 = bitcast <1 x i64> %shuffle.i.i to <2 x i32>2686 %vecinit.i = insertelement <2 x i32> undef, i32 %d, i32 02687 %vecinit1.i = insertelement <2 x i32> %vecinit.i, i32 %d, i32 12688 %vmull2.i.i = tail call <2 x i64> @llvm.aarch64.neon.smull.v2i64(<2 x i32> %1, <2 x i32> %vecinit1.i) nounwind2689 ret <2 x i64> %vmull2.i.i2690}2691 2692define <4 x i32> @vmull_high_n_u16_test(<4 x i32> %a, <8 x i16> %b, <4 x i16> %c, i32 %d) nounwind readnone optsize ssp {2693; CHECK-SD-LABEL: vmull_high_n_u16_test:2694; CHECK-SD: // %bb.0: // %entry2695; CHECK-SD-NEXT: dup v0.8h, w02696; CHECK-SD-NEXT: umull2 v0.4s, v1.8h, v0.8h2697; CHECK-SD-NEXT: ret2698;2699; CHECK-GI-LABEL: vmull_high_n_u16_test:2700; CHECK-GI: // %bb.0: // %entry2701; CHECK-GI-NEXT: mov d0, v1.d[1]2702; CHECK-GI-NEXT: dup v1.4h, w02703; CHECK-GI-NEXT: umull v0.4s, v0.4h, v1.4h2704; CHECK-GI-NEXT: ret2705entry:2706 %conv = trunc i32 %d to i162707 %0 = bitcast <8 x i16> %b to <2 x i64>2708 %shuffle.i.i = shufflevector <2 x i64> %0, <2 x i64> undef, <1 x i32> <i32 1>2709 %1 = bitcast <1 x i64> %shuffle.i.i to <4 x i16>2710 %vecinit.i = insertelement <4 x i16> undef, i16 %conv, i32 02711 %vecinit1.i = insertelement <4 x i16> %vecinit.i, i16 %conv, i32 12712 %vecinit2.i = insertelement <4 x i16> %vecinit1.i, i16 %conv, i32 22713 %vecinit3.i = insertelement <4 x i16> %vecinit2.i, i16 %conv, i32 32714 %vmull2.i.i = tail call <4 x i32> @llvm.aarch64.neon.umull.v4i32(<4 x i16> %1, <4 x i16> %vecinit3.i) nounwind2715 ret <4 x i32> %vmull2.i.i2716}2717 2718define <2 x i64> @vmull_high_n_u32_test(<2 x i64> %a, <4 x i32> %b, <2 x i32> %c, i32 %d) nounwind readnone optsize ssp {2719; CHECK-SD-LABEL: vmull_high_n_u32_test:2720; CHECK-SD: // %bb.0: // %entry2721; CHECK-SD-NEXT: dup v0.4s, w02722; CHECK-SD-NEXT: umull2 v0.2d, v1.4s, v0.4s2723; CHECK-SD-NEXT: ret2724;2725; CHECK-GI-LABEL: vmull_high_n_u32_test:2726; CHECK-GI: // %bb.0: // %entry2727; CHECK-GI-NEXT: mov d0, v1.d[1]2728; CHECK-GI-NEXT: dup v1.2s, w02729; CHECK-GI-NEXT: umull v0.2d, v0.2s, v1.2s2730; CHECK-GI-NEXT: ret2731entry:2732 %0 = bitcast <4 x i32> %b to <2 x i64>2733 %shuffle.i.i = shufflevector <2 x i64> %0, <2 x i64> undef, <1 x i32> <i32 1>2734 %1 = bitcast <1 x i64> %shuffle.i.i to <2 x i32>2735 %vecinit.i = insertelement <2 x i32> undef, i32 %d, i32 02736 %vecinit1.i = insertelement <2 x i32> %vecinit.i, i32 %d, i32 12737 %vmull2.i.i = tail call <2 x i64> @llvm.aarch64.neon.umull.v2i64(<2 x i32> %1, <2 x i32> %vecinit1.i) nounwind2738 ret <2 x i64> %vmull2.i.i2739}2740 2741define <4 x i32> @vmul_built_dup_test(<4 x i32> %a, <4 x i32> %b) {2742; CHECK-SD-LABEL: vmul_built_dup_test:2743; CHECK-SD: // %bb.0:2744; CHECK-SD-NEXT: mul v0.4s, v0.4s, v1.s[1]2745; CHECK-SD-NEXT: ret2746;2747; CHECK-GI-LABEL: vmul_built_dup_test:2748; CHECK-GI: // %bb.0:2749; CHECK-GI-NEXT: mov s1, v1.s[1]2750; CHECK-GI-NEXT: dup v1.4s, v1.s[0]2751; CHECK-GI-NEXT: mul v0.4s, v0.4s, v1.4s2752; CHECK-GI-NEXT: ret2753 %vget_lane = extractelement <4 x i32> %b, i32 12754 %vecinit.i = insertelement <4 x i32> undef, i32 %vget_lane, i32 02755 %vecinit1.i = insertelement <4 x i32> %vecinit.i, i32 %vget_lane, i32 12756 %vecinit2.i = insertelement <4 x i32> %vecinit1.i, i32 %vget_lane, i32 22757 %vecinit3.i = insertelement <4 x i32> %vecinit2.i, i32 %vget_lane, i32 32758 %prod = mul <4 x i32> %a, %vecinit3.i2759 ret <4 x i32> %prod2760}2761 2762define <4 x i16> @vmul_built_dup_fromsmall_test(<4 x i16> %a, <4 x i16> %b) {2763; CHECK-SD-LABEL: vmul_built_dup_fromsmall_test:2764; CHECK-SD: // %bb.0:2765; CHECK-SD-NEXT: // kill: def $d1 killed $d1 def $q12766; CHECK-SD-NEXT: mul v0.4h, v0.4h, v1.h[3]2767; CHECK-SD-NEXT: ret2768;2769; CHECK-GI-LABEL: vmul_built_dup_fromsmall_test:2770; CHECK-GI: // %bb.0:2771; CHECK-GI-NEXT: // kill: def $d1 killed $d1 def $q12772; CHECK-GI-NEXT: mov h1, v1.h[3]2773; CHECK-GI-NEXT: dup v1.4h, v1.h[0]2774; CHECK-GI-NEXT: mul v0.4h, v0.4h, v1.4h2775; CHECK-GI-NEXT: ret2776 %vget_lane = extractelement <4 x i16> %b, i32 32777 %vecinit.i = insertelement <4 x i16> undef, i16 %vget_lane, i32 02778 %vecinit1.i = insertelement <4 x i16> %vecinit.i, i16 %vget_lane, i32 12779 %vecinit2.i = insertelement <4 x i16> %vecinit1.i, i16 %vget_lane, i32 22780 %vecinit3.i = insertelement <4 x i16> %vecinit2.i, i16 %vget_lane, i32 32781 %prod = mul <4 x i16> %a, %vecinit3.i2782 ret <4 x i16> %prod2783}2784 2785define <8 x i16> @vmulq_built_dup_fromsmall_test(<8 x i16> %a, <4 x i16> %b) {2786; CHECK-SD-LABEL: vmulq_built_dup_fromsmall_test:2787; CHECK-SD: // %bb.0:2788; CHECK-SD-NEXT: // kill: def $d1 killed $d1 def $q12789; CHECK-SD-NEXT: mul v0.8h, v0.8h, v1.h[0]2790; CHECK-SD-NEXT: ret2791;2792; CHECK-GI-LABEL: vmulq_built_dup_fromsmall_test:2793; CHECK-GI: // %bb.0:2794; CHECK-GI-NEXT: // kill: def $d1 killed $d1 def $q12795; CHECK-GI-NEXT: dup v1.8h, v1.h[0]2796; CHECK-GI-NEXT: mul v0.8h, v0.8h, v1.8h2797; CHECK-GI-NEXT: ret2798 %vget_lane = extractelement <4 x i16> %b, i32 02799 %vecinit.i = insertelement <8 x i16> undef, i16 %vget_lane, i32 02800 %vecinit1.i = insertelement <8 x i16> %vecinit.i, i16 %vget_lane, i32 12801 %vecinit2.i = insertelement <8 x i16> %vecinit1.i, i16 %vget_lane, i32 22802 %vecinit3.i = insertelement <8 x i16> %vecinit2.i, i16 %vget_lane, i32 32803 %vecinit4.i = insertelement <8 x i16> %vecinit3.i, i16 %vget_lane, i32 42804 %vecinit5.i = insertelement <8 x i16> %vecinit4.i, i16 %vget_lane, i32 52805 %vecinit6.i = insertelement <8 x i16> %vecinit5.i, i16 %vget_lane, i32 62806 %vecinit7.i = insertelement <8 x i16> %vecinit6.i, i16 %vget_lane, i32 72807 %prod = mul <8 x i16> %a, %vecinit7.i2808 ret <8 x i16> %prod2809}2810 2811define <2 x i64> @mull_from_two_extracts(<4 x i32> %lhs, <4 x i32> %rhs) {2812; CHECK-LABEL: mull_from_two_extracts:2813; CHECK: // %bb.0:2814; CHECK-NEXT: sqdmull2 v0.2d, v0.4s, v1.4s2815; CHECK-NEXT: ret2816 %lhs.high = shufflevector <4 x i32> %lhs, <4 x i32> undef, <2 x i32> <i32 2, i32 3>2817 %rhs.high = shufflevector <4 x i32> %rhs, <4 x i32> undef, <2 x i32> <i32 2, i32 3>2818 2819 %res = tail call <2 x i64> @llvm.aarch64.neon.sqdmull.v2i64(<2 x i32> %lhs.high, <2 x i32> %rhs.high) nounwind2820 ret <2 x i64> %res2821}2822 2823define <2 x i64> @mlal_from_two_extracts(<2 x i64> %accum, <4 x i32> %lhs, <4 x i32> %rhs) {2824; CHECK-LABEL: mlal_from_two_extracts:2825; CHECK: // %bb.0:2826; CHECK-NEXT: sqdmlal2 v0.2d, v1.4s, v2.4s2827; CHECK-NEXT: ret2828 %lhs.high = shufflevector <4 x i32> %lhs, <4 x i32> undef, <2 x i32> <i32 2, i32 3>2829 %rhs.high = shufflevector <4 x i32> %rhs, <4 x i32> undef, <2 x i32> <i32 2, i32 3>2830 2831 %res = tail call <2 x i64> @llvm.aarch64.neon.sqdmull.v2i64(<2 x i32> %lhs.high, <2 x i32> %rhs.high) nounwind2832 %sum = call <2 x i64> @llvm.aarch64.neon.sqadd.v2i64(<2 x i64> %accum, <2 x i64> %res)2833 ret <2 x i64> %sum2834}2835 2836define <2 x i64> @mull_from_extract_dup_low(<4 x i32> %lhs, i32 %rhs) {2837; CHECK-LABEL: mull_from_extract_dup_low:2838; CHECK: // %bb.0:2839; CHECK-NEXT: dup v1.2s, w02840; CHECK-NEXT: sqdmull v0.2d, v0.2s, v1.2s2841; CHECK-NEXT: ret2842 %rhsvec.tmp = insertelement <2 x i32> undef, i32 %rhs, i32 02843 %rhsvec = insertelement <2 x i32> %rhsvec.tmp, i32 %rhs, i32 12844 2845 %lhs.high = shufflevector <4 x i32> %lhs, <4 x i32> undef, <2 x i32> <i32 0, i32 1>2846 2847 %res = tail call <2 x i64> @llvm.aarch64.neon.sqdmull.v2i64(<2 x i32> %lhs.high, <2 x i32> %rhsvec) nounwind2848 ret <2 x i64> %res2849}2850 2851define <2 x i64> @mull_from_extract_dup_high(<4 x i32> %lhs, i32 %rhs) {2852; CHECK-SD-LABEL: mull_from_extract_dup_high:2853; CHECK-SD: // %bb.0:2854; CHECK-SD-NEXT: dup v1.4s, w02855; CHECK-SD-NEXT: sqdmull2 v0.2d, v0.4s, v1.4s2856; CHECK-SD-NEXT: ret2857;2858; CHECK-GI-LABEL: mull_from_extract_dup_high:2859; CHECK-GI: // %bb.0:2860; CHECK-GI-NEXT: dup v1.2s, w02861; CHECK-GI-NEXT: mov d0, v0.d[1]2862; CHECK-GI-NEXT: sqdmull v0.2d, v0.2s, v1.2s2863; CHECK-GI-NEXT: ret2864 %rhsvec.tmp = insertelement <2 x i32> undef, i32 %rhs, i32 02865 %rhsvec = insertelement <2 x i32> %rhsvec.tmp, i32 %rhs, i32 12866 2867 %lhs.high = shufflevector <4 x i32> %lhs, <4 x i32> undef, <2 x i32> <i32 2, i32 3>2868 2869 %res = tail call <2 x i64> @llvm.aarch64.neon.sqdmull.v2i64(<2 x i32> %lhs.high, <2 x i32> %rhsvec) nounwind2870 ret <2 x i64> %res2871}2872 2873define <8 x i16> @pmull_from_extract_dup_low(<16 x i8> %lhs, i8 %rhs) {2874; CHECK-LABEL: pmull_from_extract_dup_low:2875; CHECK: // %bb.0:2876; CHECK-NEXT: dup v1.8b, w02877; CHECK-NEXT: pmull v0.8h, v0.8b, v1.8b2878; CHECK-NEXT: ret2879 %rhsvec.0 = insertelement <8 x i8> undef, i8 %rhs, i32 02880 %rhsvec = shufflevector <8 x i8> %rhsvec.0, <8 x i8> undef, <8 x i32> <i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0>2881 2882 %lhs.high = shufflevector <16 x i8> %lhs, <16 x i8> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>2883 2884 %res = tail call <8 x i16> @llvm.aarch64.neon.pmull.v8i16(<8 x i8> %lhs.high, <8 x i8> %rhsvec) nounwind2885 ret <8 x i16> %res2886}2887 2888define <8 x i16> @pmull_from_extract_dup_high(<16 x i8> %lhs, i8 %rhs) {2889; CHECK-SD-LABEL: pmull_from_extract_dup_high:2890; CHECK-SD: // %bb.0:2891; CHECK-SD-NEXT: dup v1.16b, w02892; CHECK-SD-NEXT: pmull2 v0.8h, v0.16b, v1.16b2893; CHECK-SD-NEXT: ret2894;2895; CHECK-GI-LABEL: pmull_from_extract_dup_high:2896; CHECK-GI: // %bb.0:2897; CHECK-GI-NEXT: dup v1.8b, w02898; CHECK-GI-NEXT: mov d0, v0.d[1]2899; CHECK-GI-NEXT: pmull v0.8h, v0.8b, v1.8b2900; CHECK-GI-NEXT: ret2901 %rhsvec.0 = insertelement <8 x i8> undef, i8 %rhs, i32 02902 %rhsvec = shufflevector <8 x i8> %rhsvec.0, <8 x i8> undef, <8 x i32> <i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0>2903 2904 %lhs.high = shufflevector <16 x i8> %lhs, <16 x i8> undef, <8 x i32> <i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>2905 2906 %res = tail call <8 x i16> @llvm.aarch64.neon.pmull.v8i16(<8 x i8> %lhs.high, <8 x i8> %rhsvec) nounwind2907 ret <8 x i16> %res2908}2909 2910define <8 x i16> @pmull_from_extract_duplane_low(<16 x i8> %lhs, <8 x i8> %rhs) {2911; CHECK-LABEL: pmull_from_extract_duplane_low:2912; CHECK: // %bb.0:2913; CHECK-NEXT: // kill: def $d1 killed $d1 def $q12914; CHECK-NEXT: dup v1.8b, v1.b[0]2915; CHECK-NEXT: pmull v0.8h, v0.8b, v1.8b2916; CHECK-NEXT: ret2917 %lhs.high = shufflevector <16 x i8> %lhs, <16 x i8> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>2918 %rhs.high = shufflevector <8 x i8> %rhs, <8 x i8> undef, <8 x i32> <i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0>2919 2920 %res = tail call <8 x i16> @llvm.aarch64.neon.pmull.v8i16(<8 x i8> %lhs.high, <8 x i8> %rhs.high) nounwind2921 ret <8 x i16> %res2922}2923 2924define <8 x i16> @pmull_from_extract_duplane_high(<16 x i8> %lhs, <8 x i8> %rhs) {2925; CHECK-SD-LABEL: pmull_from_extract_duplane_high:2926; CHECK-SD: // %bb.0:2927; CHECK-SD-NEXT: // kill: def $d1 killed $d1 def $q12928; CHECK-SD-NEXT: dup v1.16b, v1.b[0]2929; CHECK-SD-NEXT: pmull2 v0.8h, v0.16b, v1.16b2930; CHECK-SD-NEXT: ret2931;2932; CHECK-GI-LABEL: pmull_from_extract_duplane_high:2933; CHECK-GI: // %bb.0:2934; CHECK-GI-NEXT: // kill: def $d1 killed $d1 def $q12935; CHECK-GI-NEXT: mov d0, v0.d[1]2936; CHECK-GI-NEXT: dup v1.8b, v1.b[0]2937; CHECK-GI-NEXT: pmull v0.8h, v0.8b, v1.8b2938; CHECK-GI-NEXT: ret2939 %lhs.high = shufflevector <16 x i8> %lhs, <16 x i8> undef, <8 x i32> <i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>2940 %rhs.high = shufflevector <8 x i8> %rhs, <8 x i8> undef, <8 x i32> <i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0>2941 2942 %res = tail call <8 x i16> @llvm.aarch64.neon.pmull.v8i16(<8 x i8> %lhs.high, <8 x i8> %rhs.high) nounwind2943 ret <8 x i16> %res2944}2945 2946define <2 x i64> @sqdmull_from_extract_duplane_low(<4 x i32> %lhs, <4 x i32> %rhs) {2947; CHECK-LABEL: sqdmull_from_extract_duplane_low:2948; CHECK: // %bb.0:2949; CHECK-NEXT: sqdmull v0.2d, v0.2s, v1.s[0]2950; CHECK-NEXT: ret2951 %lhs.high = shufflevector <4 x i32> %lhs, <4 x i32> undef, <2 x i32> <i32 0, i32 1>2952 %rhs.high = shufflevector <4 x i32> %rhs, <4 x i32> undef, <2 x i32> <i32 0, i32 0>2953 2954 %res = tail call <2 x i64> @llvm.aarch64.neon.sqdmull.v2i64(<2 x i32> %lhs.high, <2 x i32> %rhs.high) nounwind2955 ret <2 x i64> %res2956}2957 2958define <2 x i64> @sqdmull_from_extract_duplane_high(<4 x i32> %lhs, <4 x i32> %rhs) {2959; CHECK-SD-LABEL: sqdmull_from_extract_duplane_high:2960; CHECK-SD: // %bb.0:2961; CHECK-SD-NEXT: sqdmull2 v0.2d, v0.4s, v1.s[0]2962; CHECK-SD-NEXT: ret2963;2964; CHECK-GI-LABEL: sqdmull_from_extract_duplane_high:2965; CHECK-GI: // %bb.0:2966; CHECK-GI-NEXT: mov d0, v0.d[1]2967; CHECK-GI-NEXT: sqdmull v0.2d, v0.2s, v1.s[0]2968; CHECK-GI-NEXT: ret2969 %lhs.high = shufflevector <4 x i32> %lhs, <4 x i32> undef, <2 x i32> <i32 2, i32 3>2970 %rhs.high = shufflevector <4 x i32> %rhs, <4 x i32> undef, <2 x i32> <i32 0, i32 0>2971 2972 %res = tail call <2 x i64> @llvm.aarch64.neon.sqdmull.v2i64(<2 x i32> %lhs.high, <2 x i32> %rhs.high) nounwind2973 ret <2 x i64> %res2974}2975 2976define <2 x i64> @sqdmlal_from_extract_duplane_low(<2 x i64> %accum, <4 x i32> %lhs, <4 x i32> %rhs) {2977; CHECK-LABEL: sqdmlal_from_extract_duplane_low:2978; CHECK: // %bb.0:2979; CHECK-NEXT: sqdmlal v0.2d, v1.2s, v2.s[0]2980; CHECK-NEXT: ret2981 %lhs.high = shufflevector <4 x i32> %lhs, <4 x i32> undef, <2 x i32> <i32 0, i32 1>2982 %rhs.high = shufflevector <4 x i32> %rhs, <4 x i32> undef, <2 x i32> <i32 0, i32 0>2983 2984 %res = tail call <2 x i64> @llvm.aarch64.neon.sqdmull.v2i64(<2 x i32> %lhs.high, <2 x i32> %rhs.high) nounwind2985 %sum = call <2 x i64> @llvm.aarch64.neon.sqadd.v2i64(<2 x i64> %accum, <2 x i64> %res)2986 ret <2 x i64> %sum2987}2988 2989define <2 x i64> @sqdmlal_from_extract_duplane_high(<2 x i64> %accum, <4 x i32> %lhs, <4 x i32> %rhs) {2990; CHECK-SD-LABEL: sqdmlal_from_extract_duplane_high:2991; CHECK-SD: // %bb.0:2992; CHECK-SD-NEXT: sqdmlal2 v0.2d, v1.4s, v2.s[0]2993; CHECK-SD-NEXT: ret2994;2995; CHECK-GI-LABEL: sqdmlal_from_extract_duplane_high:2996; CHECK-GI: // %bb.0:2997; CHECK-GI-NEXT: mov d1, v1.d[1]2998; CHECK-GI-NEXT: sqdmlal v0.2d, v1.2s, v2.s[0]2999; CHECK-GI-NEXT: ret3000 %lhs.high = shufflevector <4 x i32> %lhs, <4 x i32> undef, <2 x i32> <i32 2, i32 3>3001 %rhs.high = shufflevector <4 x i32> %rhs, <4 x i32> undef, <2 x i32> <i32 0, i32 0>3002 3003 %res = tail call <2 x i64> @llvm.aarch64.neon.sqdmull.v2i64(<2 x i32> %lhs.high, <2 x i32> %rhs.high) nounwind3004 %sum = call <2 x i64> @llvm.aarch64.neon.sqadd.v2i64(<2 x i64> %accum, <2 x i64> %res)3005 ret <2 x i64> %sum3006}3007 3008define <2 x i64> @umlal_from_extract_duplane_low(<2 x i64> %accum, <4 x i32> %lhs, <4 x i32> %rhs) {3009; CHECK-LABEL: umlal_from_extract_duplane_low:3010; CHECK: // %bb.0:3011; CHECK-NEXT: umlal v0.2d, v1.2s, v2.s[0]3012; CHECK-NEXT: ret3013 %lhs.high = shufflevector <4 x i32> %lhs, <4 x i32> undef, <2 x i32> <i32 0, i32 1>3014 %rhs.high = shufflevector <4 x i32> %rhs, <4 x i32> undef, <2 x i32> <i32 0, i32 0>3015 3016 %res = tail call <2 x i64> @llvm.aarch64.neon.umull.v2i64(<2 x i32> %lhs.high, <2 x i32> %rhs.high) nounwind3017 %sum = add <2 x i64> %accum, %res3018 ret <2 x i64> %sum3019}3020 3021define <2 x i64> @umlal_from_extract_duplane_high(<2 x i64> %accum, <4 x i32> %lhs, <4 x i32> %rhs) {3022; CHECK-SD-LABEL: umlal_from_extract_duplane_high:3023; CHECK-SD: // %bb.0:3024; CHECK-SD-NEXT: umlal2 v0.2d, v1.4s, v2.s[0]3025; CHECK-SD-NEXT: ret3026;3027; CHECK-GI-LABEL: umlal_from_extract_duplane_high:3028; CHECK-GI: // %bb.0:3029; CHECK-GI-NEXT: mov d1, v1.d[1]3030; CHECK-GI-NEXT: umlal v0.2d, v1.2s, v2.s[0]3031; CHECK-GI-NEXT: ret3032 %lhs.high = shufflevector <4 x i32> %lhs, <4 x i32> undef, <2 x i32> <i32 2, i32 3>3033 %rhs.high = shufflevector <4 x i32> %rhs, <4 x i32> undef, <2 x i32> <i32 0, i32 0>3034 3035 %res = tail call <2 x i64> @llvm.aarch64.neon.umull.v2i64(<2 x i32> %lhs.high, <2 x i32> %rhs.high) nounwind3036 %sum = add <2 x i64> %accum, %res3037 ret <2 x i64> %sum3038}3039 3040define float @scalar_fmla_from_extract_v4f32(float %accum, float %lhs, <4 x float> %rvec) {3041; CHECK-LABEL: scalar_fmla_from_extract_v4f32:3042; CHECK: // %bb.0:3043; CHECK-NEXT: fmla s0, s1, v2.s[3]3044; CHECK-NEXT: ret3045 %rhs = extractelement <4 x float> %rvec, i32 33046 %res = call float @llvm.fma.f32(float %lhs, float %rhs, float %accum)3047 ret float %res3048}3049 3050define float @scalar_fmla_from_extract_v2f32(float %accum, float %lhs, <2 x float> %rvec) {3051; CHECK-SD-LABEL: scalar_fmla_from_extract_v2f32:3052; CHECK-SD: // %bb.0:3053; CHECK-SD-NEXT: // kill: def $d2 killed $d2 def $q23054; CHECK-SD-NEXT: fmla s0, s1, v2.s[1]3055; CHECK-SD-NEXT: ret3056;3057; CHECK-GI-LABEL: scalar_fmla_from_extract_v2f32:3058; CHECK-GI: // %bb.0:3059; CHECK-GI-NEXT: // kill: def $d2 killed $d2 def $q23060; CHECK-GI-NEXT: mov s2, v2.s[1]3061; CHECK-GI-NEXT: fmadd s0, s1, s2, s03062; CHECK-GI-NEXT: ret3063 %rhs = extractelement <2 x float> %rvec, i32 13064 %res = call float @llvm.fma.f32(float %lhs, float %rhs, float %accum)3065 ret float %res3066}3067 3068define float @scalar_fmls_from_extract_v4f32(float %accum, float %lhs, <4 x float> %rvec) {3069; CHECK-LABEL: scalar_fmls_from_extract_v4f32:3070; CHECK: // %bb.0:3071; CHECK-NEXT: fmls s0, s1, v2.s[3]3072; CHECK-NEXT: ret3073 %rhs.scal = extractelement <4 x float> %rvec, i32 33074 %rhs = fsub float -0.0, %rhs.scal3075 %res = call float @llvm.fma.f32(float %lhs, float %rhs, float %accum)3076 ret float %res3077}3078 3079define float @scalar_fmls_from_extract_v2f32(float %accum, float %lhs, <2 x float> %rvec) {3080; CHECK-LABEL: scalar_fmls_from_extract_v2f32:3081; CHECK: // %bb.0:3082; CHECK-NEXT: // kill: def $d2 killed $d2 def $q23083; CHECK-NEXT: fmls s0, s1, v2.s[1]3084; CHECK-NEXT: ret3085 %rhs.scal = extractelement <2 x float> %rvec, i32 13086 %rhs = fsub float -0.0, %rhs.scal3087 %res = call float @llvm.fma.f32(float %lhs, float %rhs, float %accum)3088 ret float %res3089}3090 3091declare float @llvm.fma.f32(float, float, float)3092 3093define double @scalar_fmla_from_extract_v2f64(double %accum, double %lhs, <2 x double> %rvec) {3094; CHECK-LABEL: scalar_fmla_from_extract_v2f64:3095; CHECK: // %bb.0:3096; CHECK-NEXT: fmla d0, d1, v2.d[1]3097; CHECK-NEXT: ret3098 %rhs = extractelement <2 x double> %rvec, i32 13099 %res = call double @llvm.fma.f64(double %lhs, double %rhs, double %accum)3100 ret double %res3101}3102 3103define double @scalar_fmls_from_extract_v2f64(double %accum, double %lhs, <2 x double> %rvec) {3104; CHECK-LABEL: scalar_fmls_from_extract_v2f64:3105; CHECK: // %bb.0:3106; CHECK-NEXT: fmls d0, d1, v2.d[1]3107; CHECK-NEXT: ret3108 %rhs.scal = extractelement <2 x double> %rvec, i32 13109 %rhs = fsub double -0.0, %rhs.scal3110 %res = call double @llvm.fma.f64(double %lhs, double %rhs, double %accum)3111 ret double %res3112}3113 3114declare double @llvm.fma.f64(double, double, double)3115 3116define <2 x float> @fmls_with_fneg_before_extract_v2f32(<2 x float> %accum, <2 x float> %lhs, <4 x float> %rhs) {3117; CHECK-LABEL: fmls_with_fneg_before_extract_v2f32:3118; CHECK: // %bb.0:3119; CHECK-NEXT: fmls v0.2s, v1.2s, v2.s[3]3120; CHECK-NEXT: ret3121 %rhs_neg = fsub <4 x float> <float -0.0, float -0.0, float -0.0, float -0.0>, %rhs3122 %splat = shufflevector <4 x float> %rhs_neg, <4 x float> undef, <2 x i32> <i32 3, i32 3>3123 %res = call <2 x float> @llvm.fma.v2f32(<2 x float> %lhs, <2 x float> %splat, <2 x float> %accum)3124 ret <2 x float> %res3125}3126 3127define <2 x float> @fmls_with_fneg_before_extract_v2f32_1(<2 x float> %accum, <2 x float> %lhs, <2 x float> %rhs) {3128; CHECK-LABEL: fmls_with_fneg_before_extract_v2f32_1:3129; CHECK: // %bb.0:3130; CHECK-NEXT: // kill: def $d2 killed $d2 def $q23131; CHECK-NEXT: fmls v0.2s, v1.2s, v2.s[1]3132; CHECK-NEXT: ret3133 %rhs_neg = fsub <2 x float> <float -0.0, float -0.0>, %rhs3134 %splat = shufflevector <2 x float> %rhs_neg, <2 x float> undef, <2 x i32> <i32 1, i32 1>3135 %res = call <2 x float> @llvm.fma.v2f32(<2 x float> %lhs, <2 x float> %splat, <2 x float> %accum)3136 ret <2 x float> %res3137}3138 3139define <4 x float> @fmls_with_fneg_before_extract_v4f32(<4 x float> %accum, <4 x float> %lhs, <4 x float> %rhs) {3140; CHECK-LABEL: fmls_with_fneg_before_extract_v4f32:3141; CHECK: // %bb.0:3142; CHECK-NEXT: fmls v0.4s, v1.4s, v2.s[3]3143; CHECK-NEXT: ret3144 %rhs_neg = fsub <4 x float> <float -0.0, float -0.0, float -0.0, float -0.0>, %rhs3145 %splat = shufflevector <4 x float> %rhs_neg, <4 x float> undef, <4 x i32> <i32 3, i32 3, i32 3, i32 3>3146 %res = call <4 x float> @llvm.fma.v4f32(<4 x float> %lhs, <4 x float> %splat, <4 x float> %accum)3147 ret <4 x float> %res3148}3149 3150define <4 x float> @fmls_with_fneg_before_extract_v4f32_1(<4 x float> %accum, <4 x float> %lhs, <2 x float> %rhs) {3151; CHECK-LABEL: fmls_with_fneg_before_extract_v4f32_1:3152; CHECK: // %bb.0:3153; CHECK-NEXT: // kill: def $d2 killed $d2 def $q23154; CHECK-NEXT: fmls v0.4s, v1.4s, v2.s[1]3155; CHECK-NEXT: ret3156 %rhs_neg = fsub <2 x float> <float -0.0, float -0.0>, %rhs3157 %splat = shufflevector <2 x float> %rhs_neg, <2 x float> undef, <4 x i32> <i32 1, i32 1, i32 1, i32 1>3158 %res = call <4 x float> @llvm.fma.v4f32(<4 x float> %lhs, <4 x float> %splat, <4 x float> %accum)3159 ret <4 x float> %res3160}3161 3162define <2 x double> @fmls_with_fneg_before_extract_v2f64(<2 x double> %accum, <2 x double> %lhs, <2 x double> %rhs) {3163; CHECK-LABEL: fmls_with_fneg_before_extract_v2f64:3164; CHECK: // %bb.0:3165; CHECK-NEXT: fmls v0.2d, v1.2d, v2.d[1]3166; CHECK-NEXT: ret3167 %rhs_neg = fsub <2 x double> <double -0.0, double -0.0>, %rhs3168 %splat = shufflevector <2 x double> %rhs_neg, <2 x double> undef, <2 x i32> <i32 1, i32 1>3169 %res = call <2 x double> @llvm.fma.v2f64(<2 x double> %lhs, <2 x double> %splat, <2 x double> %accum)3170 ret <2 x double> %res3171}3172 3173define <1 x double> @test_fmul_v1f64(<1 x double> %L, <1 x double> %R) nounwind {3174; CHECK-LABEL: test_fmul_v1f64:3175; CHECK: // %bb.0:3176; CHECK-NEXT: fmul d0, d0, d13177; CHECK-NEXT: ret3178 %prod = fmul <1 x double> %L, %R3179 ret <1 x double> %prod3180}3181 3182define <1 x double> @test_fdiv_v1f64(<1 x double> %L, <1 x double> %R) nounwind {3183; CHECK-LABEL: test_fdiv_v1f64:3184; CHECK: // %bb.0:3185; CHECK-NEXT: fdiv d0, d0, d13186; CHECK-NEXT: ret3187 %prod = fdiv <1 x double> %L, %R3188 ret <1 x double> %prod3189}3190 3191define i32 @sqdmlal_s(i16 %A, i16 %B, i32 %C) nounwind {3192; CHECK-LABEL: sqdmlal_s:3193; CHECK: // %bb.0:3194; CHECK-NEXT: fmov s0, w03195; CHECK-NEXT: fmov s1, w13196; CHECK-NEXT: fmov s2, w23197; CHECK-NEXT: sqdmlal s2, h0, v1.h[0]3198; CHECK-NEXT: fmov w0, s23199; CHECK-NEXT: ret3200 %tmp1 = insertelement <4 x i16> undef, i16 %A, i64 03201 %tmp2 = insertelement <4 x i16> undef, i16 %B, i64 03202 %tmp3 = tail call <4 x i32> @llvm.aarch64.neon.sqdmull.v4i32(<4 x i16> %tmp1, <4 x i16> %tmp2)3203 %tmp4 = extractelement <4 x i32> %tmp3, i64 03204 %tmp5 = tail call i32 @llvm.aarch64.neon.sqadd.i32(i32 %C, i32 %tmp4)3205 ret i32 %tmp53206}3207 3208define i64 @sqdmlal_d(i32 %A, i32 %B, i64 %C) nounwind {3209; CHECK-LABEL: sqdmlal_d:3210; CHECK: // %bb.0:3211; CHECK-NEXT: fmov d0, x23212; CHECK-NEXT: fmov s1, w03213; CHECK-NEXT: fmov s2, w13214; CHECK-NEXT: sqdmlal d0, s1, s23215; CHECK-NEXT: fmov x0, d03216; CHECK-NEXT: ret3217 %tmp4 = call i64 @llvm.aarch64.neon.sqdmulls.scalar(i32 %A, i32 %B)3218 %tmp5 = call i64 @llvm.aarch64.neon.sqadd.i64(i64 %C, i64 %tmp4)3219 ret i64 %tmp53220}3221 3222define i32 @sqdmlsl_s(i16 %A, i16 %B, i32 %C) nounwind {3223; CHECK-LABEL: sqdmlsl_s:3224; CHECK: // %bb.0:3225; CHECK-NEXT: fmov s0, w03226; CHECK-NEXT: fmov s1, w13227; CHECK-NEXT: fmov s2, w23228; CHECK-NEXT: sqdmlsl s2, h0, v1.h[0]3229; CHECK-NEXT: fmov w0, s23230; CHECK-NEXT: ret3231 %tmp1 = insertelement <4 x i16> undef, i16 %A, i64 03232 %tmp2 = insertelement <4 x i16> undef, i16 %B, i64 03233 %tmp3 = tail call <4 x i32> @llvm.aarch64.neon.sqdmull.v4i32(<4 x i16> %tmp1, <4 x i16> %tmp2)3234 %tmp4 = extractelement <4 x i32> %tmp3, i64 03235 %tmp5 = tail call i32 @llvm.aarch64.neon.sqsub.i32(i32 %C, i32 %tmp4)3236 ret i32 %tmp53237}3238 3239define i64 @sqdmlsl_d(i32 %A, i32 %B, i64 %C) nounwind {3240; CHECK-LABEL: sqdmlsl_d:3241; CHECK: // %bb.0:3242; CHECK-NEXT: fmov d0, x23243; CHECK-NEXT: fmov s1, w03244; CHECK-NEXT: fmov s2, w13245; CHECK-NEXT: sqdmlsl d0, s1, s23246; CHECK-NEXT: fmov x0, d03247; CHECK-NEXT: ret3248 %tmp4 = call i64 @llvm.aarch64.neon.sqdmulls.scalar(i32 %A, i32 %B)3249 %tmp5 = call i64 @llvm.aarch64.neon.sqsub.i64(i64 %C, i64 %tmp4)3250 ret i64 %tmp53251}3252 3253define <16 x i8> @test_pmull_64(i64 %l, i64 %r) nounwind {3254; CHECK-SD-LABEL: test_pmull_64:3255; CHECK-SD: // %bb.0:3256; CHECK-SD-NEXT: fmov d0, x13257; CHECK-SD-NEXT: fmov d1, x03258; CHECK-SD-NEXT: pmull v0.1q, v1.1d, v0.1d3259; CHECK-SD-NEXT: ret3260;3261; CHECK-GI-LABEL: test_pmull_64:3262; CHECK-GI: // %bb.0:3263; CHECK-GI-NEXT: fmov d0, x03264; CHECK-GI-NEXT: fmov d1, x13265; CHECK-GI-NEXT: pmull v0.1q, v0.1d, v1.1d3266; CHECK-GI-NEXT: ret3267 %val = call <16 x i8> @llvm.aarch64.neon.pmull64(i64 %l, i64 %r)3268 ret <16 x i8> %val3269}3270 3271define <16 x i8> @test_pmull_high_64(<2 x i64> %l, <2 x i64> %r) nounwind {3272; CHECK-SD-LABEL: test_pmull_high_64:3273; CHECK-SD: // %bb.0:3274; CHECK-SD-NEXT: pmull2 v0.1q, v0.2d, v1.2d3275; CHECK-SD-NEXT: ret3276;3277; CHECK-GI-LABEL: test_pmull_high_64:3278; CHECK-GI: // %bb.0:3279; CHECK-GI-NEXT: mov d0, v0.d[1]3280; CHECK-GI-NEXT: mov d1, v1.d[1]3281; CHECK-GI-NEXT: pmull v0.1q, v0.1d, v1.1d3282; CHECK-GI-NEXT: ret3283 %l_hi = extractelement <2 x i64> %l, i32 13284 %r_hi = extractelement <2 x i64> %r, i32 13285 %val = call <16 x i8> @llvm.aarch64.neon.pmull64(i64 %l_hi, i64 %r_hi)3286 ret <16 x i8> %val3287}3288 3289define <16 x i8> @test_commutable_pmull_64(i64 %l, i64 %r) nounwind {3290; CHECK-SD-LABEL: test_commutable_pmull_64:3291; CHECK-SD: // %bb.0:3292; CHECK-SD-NEXT: fmov d0, x13293; CHECK-SD-NEXT: fmov d1, x03294; CHECK-SD-NEXT: pmull v0.1q, v1.1d, v0.1d3295; CHECK-SD-NEXT: add v0.16b, v0.16b, v0.16b3296; CHECK-SD-NEXT: ret3297;3298; CHECK-GI-LABEL: test_commutable_pmull_64:3299; CHECK-GI: // %bb.0:3300; CHECK-GI-NEXT: fmov d0, x03301; CHECK-GI-NEXT: fmov d1, x13302; CHECK-GI-NEXT: pmull v2.1q, v0.1d, v1.1d3303; CHECK-GI-NEXT: pmull v0.1q, v1.1d, v0.1d3304; CHECK-GI-NEXT: add v0.16b, v2.16b, v0.16b3305; CHECK-GI-NEXT: ret3306 %1 = call <16 x i8> @llvm.aarch64.neon.pmull64(i64 %l, i64 %r)3307 %2 = call <16 x i8> @llvm.aarch64.neon.pmull64(i64 %r, i64 %l)3308 %3 = add <16 x i8> %1, %23309 ret <16 x i8> %33310}3311 3312declare <16 x i8> @llvm.aarch64.neon.pmull64(i64, i64)3313 3314define <1 x i64> @test_mul_v1i64(<1 x i64> %lhs, <1 x i64> %rhs) nounwind {3315; CHECK-SD-LABEL: test_mul_v1i64:3316; CHECK-SD: // %bb.0:3317; CHECK-SD-NEXT: // kill: def $d1 killed $d1 def $q13318; CHECK-SD-NEXT: // kill: def $d0 killed $d0 def $q03319; CHECK-SD-NEXT: fmov x8, d13320; CHECK-SD-NEXT: fmov x9, d03321; CHECK-SD-NEXT: mul x8, x9, x83322; CHECK-SD-NEXT: fmov d0, x83323; CHECK-SD-NEXT: ret3324;3325; CHECK-GI-LABEL: test_mul_v1i64:3326; CHECK-GI: // %bb.0:3327; CHECK-GI-NEXT: fmov x8, d03328; CHECK-GI-NEXT: fmov x9, d13329; CHECK-GI-NEXT: mul x8, x8, x93330; CHECK-GI-NEXT: fmov d0, x83331; CHECK-GI-NEXT: ret3332 %prod = mul <1 x i64> %lhs, %rhs3333 ret <1 x i64> %prod3334}3335 3336define <4 x i32> @sqdmlal4s_lib(<4 x i32> %dst, <4 x i16> %v1, <4 x i16> %v2) {3337; CHECK-LABEL: sqdmlal4s_lib:3338; CHECK: // %bb.0:3339; CHECK-NEXT: sqdmlal v0.4s, v1.4h, v2.4h3340; CHECK-NEXT: ret3341 %tmp = call <4 x i32> @llvm.aarch64.neon.sqdmull.v4i32(<4 x i16> %v1, <4 x i16> %v2)3342 %sum = call <4 x i32> @llvm.sadd.sat.v4i32(<4 x i32> %dst, <4 x i32> %tmp)3343 ret <4 x i32> %sum3344}3345 3346define <2 x i64> @sqdmlal2d_lib(<2 x i64> %dst, <2 x i32> %v1, <2 x i32> %v2) {3347; CHECK-LABEL: sqdmlal2d_lib:3348; CHECK: // %bb.0:3349; CHECK-NEXT: sqdmlal v0.2d, v1.2s, v2.2s3350; CHECK-NEXT: ret3351 %tmp = call <2 x i64> @llvm.aarch64.neon.sqdmull.v2i64(<2 x i32> %v1, <2 x i32> %v2)3352 %sum = call <2 x i64> @llvm.sadd.sat.v2i64(<2 x i64> %dst, <2 x i64> %tmp)3353 ret <2 x i64> %sum3354}3355 3356define <4 x i32> @sqdmlal2_4s_lib(<4 x i32> %dst, <8 x i16> %v1, <8 x i16> %v2) {3357; CHECK-LABEL: sqdmlal2_4s_lib:3358; CHECK: // %bb.0:3359; CHECK-NEXT: sqdmlal2 v0.4s, v1.8h, v2.8h3360; CHECK-NEXT: ret3361 %tmp0 = shufflevector <8 x i16> %v1, <8 x i16> poison, <4 x i32> <i32 4, i32 5, i32 6, i32 7>3362 %tmp1 = shufflevector <8 x i16> %v2, <8 x i16> poison, <4 x i32> <i32 4, i32 5, i32 6, i32 7>3363 %tmp2 = call <4 x i32> @llvm.aarch64.neon.sqdmull.v4i32(<4 x i16> %tmp0, <4 x i16> %tmp1)3364 %sum = call <4 x i32> @llvm.sadd.sat.v4i32(<4 x i32> %dst, <4 x i32> %tmp2)3365 ret <4 x i32> %sum3366}3367 3368define <2 x i64> @sqdmlal2_2d_lib(<2 x i64> %dst, <4 x i32> %v1, <4 x i32> %v2) {3369; CHECK-LABEL: sqdmlal2_2d_lib:3370; CHECK: // %bb.0:3371; CHECK-NEXT: sqdmlal2 v0.2d, v1.4s, v2.4s3372; CHECK-NEXT: ret3373 %tmp0 = shufflevector <4 x i32> %v1, <4 x i32> poison, <2 x i32> <i32 2, i32 3>3374 %tmp1 = shufflevector <4 x i32> %v2, <4 x i32> poison, <2 x i32> <i32 2, i32 3>3375 %tmp2 = call <2 x i64> @llvm.aarch64.neon.sqdmull.v2i64(<2 x i32> %tmp0, <2 x i32> %tmp1)3376 %sum = call <2 x i64> @llvm.sadd.sat.v2i64(<2 x i64> %dst, <2 x i64> %tmp2)3377 ret <2 x i64> %sum3378}3379 3380define <4 x i32> @sqdmlal_lane_4s_lib(<4 x i32> %dst, <4 x i16> %v1, <4 x i16> %v2) {3381; CHECK-LABEL: sqdmlal_lane_4s_lib:3382; CHECK: // %bb.0:3383; CHECK-NEXT: // kill: def $d2 killed $d2 def $q23384; CHECK-NEXT: sqdmlal v0.4s, v1.4h, v2.h[3]3385; CHECK-NEXT: ret3386 %tmp0 = shufflevector <4 x i16> %v2, <4 x i16> poison, <4 x i32> <i32 3, i32 3, i32 3, i32 3>3387 %tmp1 = call <4 x i32> @llvm.aarch64.neon.sqdmull.v4i32(<4 x i16> %v1, <4 x i16> %tmp0)3388 %sum = call <4 x i32> @llvm.sadd.sat.v4i32(<4 x i32> %dst, <4 x i32> %tmp1)3389 ret <4 x i32> %sum3390}3391 3392define <2 x i64> @sqdmlal_lane_2d_lib(<2 x i64> %dst, <2 x i32> %v1, <2 x i32> %v2) {3393; CHECK-LABEL: sqdmlal_lane_2d_lib:3394; CHECK: // %bb.0:3395; CHECK-NEXT: // kill: def $d2 killed $d2 def $q23396; CHECK-NEXT: sqdmlal v0.2d, v1.2s, v2.s[1]3397; CHECK-NEXT: ret3398 %tmp0 = shufflevector <2 x i32> %v2, <2 x i32> poison, <2 x i32> <i32 1, i32 1>3399 %tmp1 = call <2 x i64> @llvm.aarch64.neon.sqdmull.v2i64(<2 x i32> %v1, <2 x i32> %tmp0)3400 %sum = call <2 x i64> @llvm.sadd.sat.v2i64(<2 x i64> %dst, <2 x i64> %tmp1)3401 ret <2 x i64> %sum3402}3403 3404define <4 x i32> @sqdmlal2_lane_4s_lib(<4 x i32> %dst, <8 x i16> %v1, <8 x i16> %v2) {3405; CHECK-SD-LABEL: sqdmlal2_lane_4s_lib:3406; CHECK-SD: // %bb.0:3407; CHECK-SD-NEXT: sqdmlal2 v0.4s, v1.8h, v2.h[7]3408; CHECK-SD-NEXT: ret3409;3410; CHECK-GI-LABEL: sqdmlal2_lane_4s_lib:3411; CHECK-GI: // %bb.0:3412; CHECK-GI-NEXT: mov d1, v1.d[1]3413; CHECK-GI-NEXT: sqdmlal v0.4s, v1.4h, v2.h[7]3414; CHECK-GI-NEXT: ret3415 %tmp0 = shufflevector <8 x i16> %v1, <8 x i16> poison, <4 x i32> <i32 4, i32 5, i32 6, i32 7>3416 %tmp1 = shufflevector <8 x i16> %v2, <8 x i16> poison, <4 x i32> <i32 7, i32 7, i32 7, i32 7>3417 %tmp2 = call <4 x i32> @llvm.aarch64.neon.sqdmull.v4i32(<4 x i16> %tmp0, <4 x i16> %tmp1)3418 %sum = call <4 x i32> @llvm.sadd.sat.v4i32(<4 x i32> %dst, <4 x i32> %tmp2)3419 ret <4 x i32> %sum3420}3421 3422define <2 x i64> @sqdmlal2_lane_2d_lib(<2 x i64> %dst, <4 x i32> %v1, <4 x i32> %v2) {3423; CHECK-SD-LABEL: sqdmlal2_lane_2d_lib:3424; CHECK-SD: // %bb.0:3425; CHECK-SD-NEXT: sqdmlal2 v0.2d, v1.4s, v2.s[1]3426; CHECK-SD-NEXT: ret3427;3428; CHECK-GI-LABEL: sqdmlal2_lane_2d_lib:3429; CHECK-GI: // %bb.0:3430; CHECK-GI-NEXT: mov d1, v1.d[1]3431; CHECK-GI-NEXT: sqdmlal v0.2d, v1.2s, v2.s[1]3432; CHECK-GI-NEXT: ret3433 %tmp0 = shufflevector <4 x i32> %v1, <4 x i32> poison, <2 x i32> <i32 2, i32 3>3434 %tmp1 = shufflevector <4 x i32> %v2, <4 x i32> poison, <2 x i32> <i32 1, i32 1>3435 %tmp2 = call <2 x i64> @llvm.aarch64.neon.sqdmull.v2i64(<2 x i32> %tmp0, <2 x i32> %tmp1)3436 %sum = call <2 x i64> @llvm.sadd.sat.v2i64(<2 x i64> %dst, <2 x i64> %tmp2)3437 ret <2 x i64> %sum3438}3439 3440define <4 x i32> @sqdmlsl4s_lib(<4 x i32> %dst, <4 x i16> %v1, <4 x i16> %v2) {3441; CHECK-LABEL: sqdmlsl4s_lib:3442; CHECK: // %bb.0:3443; CHECK-NEXT: sqdmlsl v0.4s, v1.4h, v2.4h3444; CHECK-NEXT: ret3445 %tmp = call <4 x i32> @llvm.aarch64.neon.sqdmull.v4i32(<4 x i16> %v1, <4 x i16> %v2)3446 %sum = call <4 x i32> @llvm.ssub.sat.v4i32(<4 x i32> %dst, <4 x i32> %tmp)3447 ret <4 x i32> %sum3448}3449 3450define <2 x i64> @sqdmlsl2d_lib(<2 x i64> %dst, <2 x i32> %v1, <2 x i32> %v2) {3451; CHECK-LABEL: sqdmlsl2d_lib:3452; CHECK: // %bb.0:3453; CHECK-NEXT: sqdmlsl v0.2d, v1.2s, v2.2s3454; CHECK-NEXT: ret3455 %tmp = call <2 x i64> @llvm.aarch64.neon.sqdmull.v2i64(<2 x i32> %v1, <2 x i32> %v2)3456 %sum = call <2 x i64> @llvm.ssub.sat.v2i64(<2 x i64> %dst, <2 x i64> %tmp)3457 ret <2 x i64> %sum3458}3459 3460define <4 x i32> @sqdmlsl2_4s_lib(<4 x i32> %dst, <8 x i16> %v1, <8 x i16> %v2) {3461; CHECK-LABEL: sqdmlsl2_4s_lib:3462; CHECK: // %bb.0:3463; CHECK-NEXT: sqdmlsl2 v0.4s, v1.8h, v2.8h3464; CHECK-NEXT: ret3465 %tmp0 = shufflevector <8 x i16> %v1, <8 x i16> poison, <4 x i32> <i32 4, i32 5, i32 6, i32 7>3466 %tmp1 = shufflevector <8 x i16> %v2, <8 x i16> poison, <4 x i32> <i32 4, i32 5, i32 6, i32 7>3467 %tmp2 = call <4 x i32> @llvm.aarch64.neon.sqdmull.v4i32(<4 x i16> %tmp0, <4 x i16> %tmp1)3468 %sum = call <4 x i32> @llvm.ssub.sat.v4i32(<4 x i32> %dst, <4 x i32> %tmp2)3469 ret <4 x i32> %sum3470}3471 3472define <2 x i64> @sqdmlsl2_2d_lib(<2 x i64> %dst, <4 x i32> %v1, <4 x i32> %v2) {3473; CHECK-LABEL: sqdmlsl2_2d_lib:3474; CHECK: // %bb.0:3475; CHECK-NEXT: sqdmlsl2 v0.2d, v1.4s, v2.4s3476; CHECK-NEXT: ret3477 %tmp0 = shufflevector <4 x i32> %v1, <4 x i32> poison, <2 x i32> <i32 2, i32 3>3478 %tmp1 = shufflevector <4 x i32> %v2, <4 x i32> poison, <2 x i32> <i32 2, i32 3>3479 %tmp2 = call <2 x i64> @llvm.aarch64.neon.sqdmull.v2i64(<2 x i32> %tmp0, <2 x i32> %tmp1)3480 %sum = call <2 x i64> @llvm.ssub.sat.v2i64(<2 x i64> %dst, <2 x i64> %tmp2)3481 ret <2 x i64> %sum3482}3483 3484define <4 x i32> @sqdmlsl_lane_4s_lib(<4 x i32> %dst, <4 x i16> %v1, <4 x i16> %v2) {3485; CHECK-LABEL: sqdmlsl_lane_4s_lib:3486; CHECK: // %bb.0:3487; CHECK-NEXT: // kill: def $d2 killed $d2 def $q23488; CHECK-NEXT: sqdmlsl v0.4s, v1.4h, v2.h[3]3489; CHECK-NEXT: ret3490 %tmp0 = shufflevector <4 x i16> %v2, <4 x i16> poison, <4 x i32> <i32 3, i32 3, i32 3, i32 3>3491 %tmp1 = call <4 x i32> @llvm.aarch64.neon.sqdmull.v4i32(<4 x i16> %v1, <4 x i16> %tmp0)3492 %sum = call <4 x i32> @llvm.ssub.sat.v4i32(<4 x i32> %dst, <4 x i32> %tmp1)3493 ret <4 x i32> %sum3494}3495 3496define <2 x i64> @sqdmlsl_lane_2d_lib(<2 x i64> %dst, <2 x i32> %v1, <2 x i32> %v2) {3497; CHECK-LABEL: sqdmlsl_lane_2d_lib:3498; CHECK: // %bb.0:3499; CHECK-NEXT: // kill: def $d2 killed $d2 def $q23500; CHECK-NEXT: sqdmlsl v0.2d, v1.2s, v2.s[1]3501; CHECK-NEXT: ret3502 %tmp0 = shufflevector <2 x i32> %v2, <2 x i32> poison, <2 x i32> <i32 1, i32 1>3503 %tmp1 = call <2 x i64> @llvm.aarch64.neon.sqdmull.v2i64(<2 x i32> %v1, <2 x i32> %tmp0)3504 %sum = call <2 x i64> @llvm.ssub.sat.v2i64(<2 x i64> %dst, <2 x i64> %tmp1)3505 ret <2 x i64> %sum3506}3507 3508define <4 x i32> @sqdmlsl2_lane_4s_lib(<4 x i32> %dst, <8 x i16> %v1, <8 x i16> %v2) {3509; CHECK-SD-LABEL: sqdmlsl2_lane_4s_lib:3510; CHECK-SD: // %bb.0:3511; CHECK-SD-NEXT: sqdmlsl2 v0.4s, v1.8h, v2.h[7]3512; CHECK-SD-NEXT: ret3513;3514; CHECK-GI-LABEL: sqdmlsl2_lane_4s_lib:3515; CHECK-GI: // %bb.0:3516; CHECK-GI-NEXT: mov d1, v1.d[1]3517; CHECK-GI-NEXT: sqdmlsl v0.4s, v1.4h, v2.h[7]3518; CHECK-GI-NEXT: ret3519 %tmp0 = shufflevector <8 x i16> %v1, <8 x i16> poison, <4 x i32> <i32 4, i32 5, i32 6, i32 7>3520 %tmp1 = shufflevector <8 x i16> %v2, <8 x i16> poison, <4 x i32> <i32 7, i32 7, i32 7, i32 7>3521 %tmp2 = call <4 x i32> @llvm.aarch64.neon.sqdmull.v4i32(<4 x i16> %tmp0, <4 x i16> %tmp1)3522 %sum = call <4 x i32> @llvm.ssub.sat.v4i32(<4 x i32> %dst, <4 x i32> %tmp2)3523 ret <4 x i32> %sum3524}3525 3526define <2 x i64> @sqdmlsl2_lane_2d_lib(<2 x i64> %dst, <4 x i32> %v1, <4 x i32> %v2) {3527; CHECK-SD-LABEL: sqdmlsl2_lane_2d_lib:3528; CHECK-SD: // %bb.0:3529; CHECK-SD-NEXT: sqdmlsl2 v0.2d, v1.4s, v2.s[1]3530; CHECK-SD-NEXT: ret3531;3532; CHECK-GI-LABEL: sqdmlsl2_lane_2d_lib:3533; CHECK-GI: // %bb.0:3534; CHECK-GI-NEXT: mov d1, v1.d[1]3535; CHECK-GI-NEXT: sqdmlsl v0.2d, v1.2s, v2.s[1]3536; CHECK-GI-NEXT: ret3537 %tmp0 = shufflevector <4 x i32> %v1, <4 x i32> poison, <2 x i32> <i32 2, i32 3>3538 %tmp1 = shufflevector <4 x i32> %v2, <4 x i32> poison, <2 x i32> <i32 1, i32 1>3539 %tmp2 = call <2 x i64> @llvm.aarch64.neon.sqdmull.v2i64(<2 x i32> %tmp0, <2 x i32> %tmp1)3540 %sum = call <2 x i64> @llvm.ssub.sat.v2i64(<2 x i64> %dst, <2 x i64> %tmp2)3541 ret <2 x i64> %sum3542}3543