3002 lines · plain
1; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py2; RUN: llc -mtriple=aarch64-none-linux-gnu -mattr=+neon < %s -o - | FileCheck %s --check-prefixes=CHECK,CHECK-NEON3; RUN: llc -mtriple=aarch64-none-linux-gnu -mattr=+sve < %s -o - | FileCheck %s --check-prefixes=CHECK,CHECK-SVE4; RUN: llc -mtriple=aarch64 -global-isel -verify-machineinstrs %s -o - | FileCheck %s --check-prefixes=CHECK,CHECK-GI5 6define <8 x i16> @smull_v8i8_v8i16(ptr %A, ptr %B) nounwind {7; CHECK-LABEL: smull_v8i8_v8i16:8; CHECK: // %bb.0:9; CHECK-NEXT: ldr d0, [x0]10; CHECK-NEXT: ldr d1, [x1]11; CHECK-NEXT: smull v0.8h, v0.8b, v1.8b12; CHECK-NEXT: ret13 %tmp1 = load <8 x i8>, ptr %A14 %tmp2 = load <8 x i8>, ptr %B15 %tmp3 = sext <8 x i8> %tmp1 to <8 x i16>16 %tmp4 = sext <8 x i8> %tmp2 to <8 x i16>17 %tmp5 = mul <8 x i16> %tmp3, %tmp418 ret <8 x i16> %tmp519}20 21define <4 x i32> @smull_v4i16_v4i32(ptr %A, ptr %B) nounwind {22; CHECK-LABEL: smull_v4i16_v4i32:23; CHECK: // %bb.0:24; CHECK-NEXT: ldr d0, [x0]25; CHECK-NEXT: ldr d1, [x1]26; CHECK-NEXT: smull v0.4s, v0.4h, v1.4h27; CHECK-NEXT: ret28 %tmp1 = load <4 x i16>, ptr %A29 %tmp2 = load <4 x i16>, ptr %B30 %tmp3 = sext <4 x i16> %tmp1 to <4 x i32>31 %tmp4 = sext <4 x i16> %tmp2 to <4 x i32>32 %tmp5 = mul <4 x i32> %tmp3, %tmp433 ret <4 x i32> %tmp534}35 36define <2 x i64> @smull_v2i32_v2i64(ptr %A, ptr %B) nounwind {37; CHECK-LABEL: smull_v2i32_v2i64:38; CHECK: // %bb.0:39; CHECK-NEXT: ldr d0, [x0]40; CHECK-NEXT: ldr d1, [x1]41; CHECK-NEXT: smull v0.2d, v0.2s, v1.2s42; CHECK-NEXT: ret43 %tmp1 = load <2 x i32>, ptr %A44 %tmp2 = load <2 x i32>, ptr %B45 %tmp3 = sext <2 x i32> %tmp1 to <2 x i64>46 %tmp4 = sext <2 x i32> %tmp2 to <2 x i64>47 %tmp5 = mul <2 x i64> %tmp3, %tmp448 ret <2 x i64> %tmp549}50 51define <8 x i32> @smull_zext_v8i8_v8i32(ptr %A, ptr %B) nounwind {52; CHECK-NEON-LABEL: smull_zext_v8i8_v8i32:53; CHECK-NEON: // %bb.0:54; CHECK-NEON-NEXT: ldr d0, [x0]55; CHECK-NEON-NEXT: ldr q2, [x1]56; CHECK-NEON-NEXT: ushll v0.8h, v0.8b, #057; CHECK-NEON-NEXT: smull2 v1.4s, v0.8h, v2.8h58; CHECK-NEON-NEXT: smull v0.4s, v0.4h, v2.4h59; CHECK-NEON-NEXT: ret60;61; CHECK-SVE-LABEL: smull_zext_v8i8_v8i32:62; CHECK-SVE: // %bb.0:63; CHECK-SVE-NEXT: ldr d0, [x0]64; CHECK-SVE-NEXT: ldr q2, [x1]65; CHECK-SVE-NEXT: ushll v0.8h, v0.8b, #066; CHECK-SVE-NEXT: smull2 v1.4s, v0.8h, v2.8h67; CHECK-SVE-NEXT: smull v0.4s, v0.4h, v2.4h68; CHECK-SVE-NEXT: ret69;70; CHECK-GI-LABEL: smull_zext_v8i8_v8i32:71; CHECK-GI: // %bb.0:72; CHECK-GI-NEXT: ldr d0, [x0]73; CHECK-GI-NEXT: ldr q2, [x1]74; CHECK-GI-NEXT: ushll v1.8h, v0.8b, #075; CHECK-GI-NEXT: smull v0.4s, v1.4h, v2.4h76; CHECK-GI-NEXT: smull2 v1.4s, v1.8h, v2.8h77; CHECK-GI-NEXT: ret78 %load.A = load <8 x i8>, ptr %A79 %load.B = load <8 x i16>, ptr %B80 %zext.A = zext <8 x i8> %load.A to <8 x i32>81 %sext.B = sext <8 x i16> %load.B to <8 x i32>82 %res = mul <8 x i32> %zext.A, %sext.B83 ret <8 x i32> %res84}85 86define <8 x i32> @smull_zext_v8i8_v8i32_sext_first_operand(ptr %A, ptr %B) nounwind {87; CHECK-NEON-LABEL: smull_zext_v8i8_v8i32_sext_first_operand:88; CHECK-NEON: // %bb.0:89; CHECK-NEON-NEXT: ldr d0, [x1]90; CHECK-NEON-NEXT: ldr q2, [x0]91; CHECK-NEON-NEXT: ushll v0.8h, v0.8b, #092; CHECK-NEON-NEXT: smull2 v1.4s, v2.8h, v0.8h93; CHECK-NEON-NEXT: smull v0.4s, v2.4h, v0.4h94; CHECK-NEON-NEXT: ret95;96; CHECK-SVE-LABEL: smull_zext_v8i8_v8i32_sext_first_operand:97; CHECK-SVE: // %bb.0:98; CHECK-SVE-NEXT: ldr d0, [x1]99; CHECK-SVE-NEXT: ldr q2, [x0]100; CHECK-SVE-NEXT: ushll v0.8h, v0.8b, #0101; CHECK-SVE-NEXT: smull2 v1.4s, v2.8h, v0.8h102; CHECK-SVE-NEXT: smull v0.4s, v2.4h, v0.4h103; CHECK-SVE-NEXT: ret104;105; CHECK-GI-LABEL: smull_zext_v8i8_v8i32_sext_first_operand:106; CHECK-GI: // %bb.0:107; CHECK-GI-NEXT: ldr d0, [x1]108; CHECK-GI-NEXT: ldr q2, [x0]109; CHECK-GI-NEXT: ushll v1.8h, v0.8b, #0110; CHECK-GI-NEXT: smull v0.4s, v2.4h, v1.4h111; CHECK-GI-NEXT: smull2 v1.4s, v2.8h, v1.8h112; CHECK-GI-NEXT: ret113 %load.A = load <8 x i16>, ptr %A114 %load.B = load <8 x i8>, ptr %B115 %sext.A = sext <8 x i16> %load.A to <8 x i32>116 %zext.B = zext <8 x i8> %load.B to <8 x i32>117 %res = mul <8 x i32> %sext.A, %zext.B118 ret <8 x i32> %res119}120 121define <8 x i32> @smull_zext_v8i8_v8i32_top_bit_is_1(ptr %A, ptr %B) nounwind {122; CHECK-NEON-LABEL: smull_zext_v8i8_v8i32_top_bit_is_1:123; CHECK-NEON: // %bb.0:124; CHECK-NEON-NEXT: ldr q0, [x0]125; CHECK-NEON-NEXT: ldr q1, [x1]126; CHECK-NEON-NEXT: orr v0.8h, #128, lsl #8127; CHECK-NEON-NEXT: sshll v3.4s, v1.4h, #0128; CHECK-NEON-NEXT: sshll2 v1.4s, v1.8h, #0129; CHECK-NEON-NEXT: ushll v2.4s, v0.4h, #0130; CHECK-NEON-NEXT: ushll2 v0.4s, v0.8h, #0131; CHECK-NEON-NEXT: mul v1.4s, v0.4s, v1.4s132; CHECK-NEON-NEXT: mul v0.4s, v2.4s, v3.4s133; CHECK-NEON-NEXT: ret134;135; CHECK-SVE-LABEL: smull_zext_v8i8_v8i32_top_bit_is_1:136; CHECK-SVE: // %bb.0:137; CHECK-SVE-NEXT: ldr q0, [x0]138; CHECK-SVE-NEXT: ldr q1, [x1]139; CHECK-SVE-NEXT: orr v0.8h, #128, lsl #8140; CHECK-SVE-NEXT: sshll v3.4s, v1.4h, #0141; CHECK-SVE-NEXT: sshll2 v1.4s, v1.8h, #0142; CHECK-SVE-NEXT: ushll v2.4s, v0.4h, #0143; CHECK-SVE-NEXT: ushll2 v0.4s, v0.8h, #0144; CHECK-SVE-NEXT: mul v1.4s, v0.4s, v1.4s145; CHECK-SVE-NEXT: mul v0.4s, v2.4s, v3.4s146; CHECK-SVE-NEXT: ret147;148; CHECK-GI-LABEL: smull_zext_v8i8_v8i32_top_bit_is_1:149; CHECK-GI: // %bb.0:150; CHECK-GI-NEXT: movi v0.8h, #128, lsl #8151; CHECK-GI-NEXT: ldr q1, [x0]152; CHECK-GI-NEXT: orr v0.16b, v1.16b, v0.16b153; CHECK-GI-NEXT: ldr q1, [x1]154; CHECK-GI-NEXT: ushll v2.4s, v0.4h, #0155; CHECK-GI-NEXT: ushll2 v3.4s, v0.8h, #0156; CHECK-GI-NEXT: sshll v0.4s, v1.4h, #0157; CHECK-GI-NEXT: sshll2 v1.4s, v1.8h, #0158; CHECK-GI-NEXT: mul v0.4s, v2.4s, v0.4s159; CHECK-GI-NEXT: mul v1.4s, v3.4s, v1.4s160; CHECK-GI-NEXT: ret161 %load.A = load <8 x i16>, ptr %A162 %or.A = or <8 x i16> %load.A, <i16 u0x8000, i16 u0x8000, i16 u0x8000, i16 u0x8000, i16 u0x8000, i16 u0x8000, i16 u0x8000, i16 u0x8000>163 %load.B = load <8 x i16>, ptr %B164 %zext.A = zext <8 x i16> %or.A to <8 x i32>165 %sext.B = sext <8 x i16> %load.B to <8 x i32>166 %res = mul <8 x i32> %zext.A, %sext.B167 ret <8 x i32> %res168}169 170define <4 x i32> @smull_zext_v4i16_v4i32(ptr %A, ptr %B) nounwind {171; CHECK-NEON-LABEL: smull_zext_v4i16_v4i32:172; CHECK-NEON: // %bb.0:173; CHECK-NEON-NEXT: ldr s0, [x0]174; CHECK-NEON-NEXT: ldr d1, [x1]175; CHECK-NEON-NEXT: ushll v0.8h, v0.8b, #0176; CHECK-NEON-NEXT: smull v0.4s, v0.4h, v1.4h177; CHECK-NEON-NEXT: ret178;179; CHECK-SVE-LABEL: smull_zext_v4i16_v4i32:180; CHECK-SVE: // %bb.0:181; CHECK-SVE-NEXT: ldr s0, [x0]182; CHECK-SVE-NEXT: ldr d1, [x1]183; CHECK-SVE-NEXT: ushll v0.8h, v0.8b, #0184; CHECK-SVE-NEXT: smull v0.4s, v0.4h, v1.4h185; CHECK-SVE-NEXT: ret186;187; CHECK-GI-LABEL: smull_zext_v4i16_v4i32:188; CHECK-GI: // %bb.0:189; CHECK-GI-NEXT: ldr w8, [x0]190; CHECK-GI-NEXT: fmov s0, w8191; CHECK-GI-NEXT: uxtb w8, w8192; CHECK-GI-NEXT: mov b1, v0.b[2]193; CHECK-GI-NEXT: mov b2, v0.b[1]194; CHECK-GI-NEXT: mov b3, v0.b[3]195; CHECK-GI-NEXT: fmov s0, w8196; CHECK-GI-NEXT: fmov w9, s1197; CHECK-GI-NEXT: fmov w10, s2198; CHECK-GI-NEXT: fmov w11, s3199; CHECK-GI-NEXT: uxtb w9, w9200; CHECK-GI-NEXT: uxtb w10, w10201; CHECK-GI-NEXT: uxtb w11, w11202; CHECK-GI-NEXT: fmov s1, w9203; CHECK-GI-NEXT: mov v0.h[1], w10204; CHECK-GI-NEXT: mov v1.h[1], w11205; CHECK-GI-NEXT: ushll v0.4s, v0.4h, #0206; CHECK-GI-NEXT: ushll v1.4s, v1.4h, #0207; CHECK-GI-NEXT: uzp1 v0.4h, v0.4h, v1.4h208; CHECK-GI-NEXT: ldr d1, [x1]209; CHECK-GI-NEXT: smull v0.4s, v0.4h, v1.4h210; CHECK-GI-NEXT: ret211 %load.A = load <4 x i8>, ptr %A212 %load.B = load <4 x i16>, ptr %B213 %zext.A = zext <4 x i8> %load.A to <4 x i32>214 %sext.B = sext <4 x i16> %load.B to <4 x i32>215 %res = mul <4 x i32> %zext.A, %sext.B216 ret <4 x i32> %res217}218 219define <2 x i64> @smull_zext_v2i32_v2i64(ptr %A, ptr %B) nounwind {220; CHECK-NEON-LABEL: smull_zext_v2i32_v2i64:221; CHECK-NEON: // %bb.0:222; CHECK-NEON-NEXT: ldr s0, [x0]223; CHECK-NEON-NEXT: ldr d1, [x1]224; CHECK-NEON-NEXT: ushll v0.4s, v0.4h, #0225; CHECK-NEON-NEXT: smull v0.2d, v0.2s, v1.2s226; CHECK-NEON-NEXT: ret227;228; CHECK-SVE-LABEL: smull_zext_v2i32_v2i64:229; CHECK-SVE: // %bb.0:230; CHECK-SVE-NEXT: ldr s0, [x0]231; CHECK-SVE-NEXT: ldr d1, [x1]232; CHECK-SVE-NEXT: ushll v0.4s, v0.4h, #0233; CHECK-SVE-NEXT: smull v0.2d, v0.2s, v1.2s234; CHECK-SVE-NEXT: ret235;236; CHECK-GI-LABEL: smull_zext_v2i32_v2i64:237; CHECK-GI: // %bb.0:238; CHECK-GI-NEXT: ld1 { v1.h }[0], [x0]239; CHECK-GI-NEXT: ldr h2, [x0, #2]240; CHECK-GI-NEXT: movi d0, #0x00ffff0000ffff241; CHECK-GI-NEXT: mov v1.s[1], v2.s[0]242; CHECK-GI-NEXT: and v0.8b, v1.8b, v0.8b243; CHECK-GI-NEXT: mov v1.s[0], v0.s[0]244; CHECK-GI-NEXT: mov v1.s[1], v0.s[1]245; CHECK-GI-NEXT: ldr d0, [x1]246; CHECK-GI-NEXT: smull v0.2d, v1.2s, v0.2s247; CHECK-GI-NEXT: ret248 %load.A = load <2 x i16>, ptr %A249 %load.B = load <2 x i32>, ptr %B250 %zext.A = zext <2 x i16> %load.A to <2 x i64>251 %sext.B = sext <2 x i32> %load.B to <2 x i64>252 %res = mul <2 x i64> %zext.A, %sext.B253 ret <2 x i64> %res254}255 256define <2 x i64> @smull_zext_and_v2i32_v2i64(ptr %A, ptr %B) nounwind {257; CHECK-NEON-LABEL: smull_zext_and_v2i32_v2i64:258; CHECK-NEON: // %bb.0:259; CHECK-NEON-NEXT: ldr d0, [x0]260; CHECK-NEON-NEXT: ldr d1, [x1]261; CHECK-NEON-NEXT: bic v0.2s, #128, lsl #24262; CHECK-NEON-NEXT: smull v0.2d, v0.2s, v1.2s263; CHECK-NEON-NEXT: ret264;265; CHECK-SVE-LABEL: smull_zext_and_v2i32_v2i64:266; CHECK-SVE: // %bb.0:267; CHECK-SVE-NEXT: ldr d0, [x0]268; CHECK-SVE-NEXT: ldr d1, [x1]269; CHECK-SVE-NEXT: bic v0.2s, #128, lsl #24270; CHECK-SVE-NEXT: smull v0.2d, v0.2s, v1.2s271; CHECK-SVE-NEXT: ret272;273; CHECK-GI-LABEL: smull_zext_and_v2i32_v2i64:274; CHECK-GI: // %bb.0:275; CHECK-GI-NEXT: mvni v0.2s, #128, lsl #24276; CHECK-GI-NEXT: ldr d1, [x0]277; CHECK-GI-NEXT: and v0.8b, v1.8b, v0.8b278; CHECK-GI-NEXT: ldr d1, [x1]279; CHECK-GI-NEXT: smull v0.2d, v0.2s, v1.2s280; CHECK-GI-NEXT: ret281 %load.A = load <2 x i32>, ptr %A282 %and.A = and <2 x i32> %load.A, <i32 u0x7FFFFFFF, i32 u0x7FFFFFFF>283 %load.B = load <2 x i32>, ptr %B284 %zext.A = zext <2 x i32> %and.A to <2 x i64>285 %sext.B = sext <2 x i32> %load.B to <2 x i64>286 %res = mul <2 x i64> %zext.A, %sext.B287 ret <2 x i64> %res288}289 290define <8 x i16> @umull_v8i8_v8i16(ptr %A, ptr %B) nounwind {291; CHECK-LABEL: umull_v8i8_v8i16:292; CHECK: // %bb.0:293; CHECK-NEXT: ldr d0, [x0]294; CHECK-NEXT: ldr d1, [x1]295; CHECK-NEXT: umull v0.8h, v0.8b, v1.8b296; CHECK-NEXT: ret297 %tmp1 = load <8 x i8>, ptr %A298 %tmp2 = load <8 x i8>, ptr %B299 %tmp3 = zext <8 x i8> %tmp1 to <8 x i16>300 %tmp4 = zext <8 x i8> %tmp2 to <8 x i16>301 %tmp5 = mul <8 x i16> %tmp3, %tmp4302 ret <8 x i16> %tmp5303}304 305define <4 x i32> @umull_v4i16_v4i32(ptr %A, ptr %B) nounwind {306; CHECK-LABEL: umull_v4i16_v4i32:307; CHECK: // %bb.0:308; CHECK-NEXT: ldr d0, [x0]309; CHECK-NEXT: ldr d1, [x1]310; CHECK-NEXT: umull v0.4s, v0.4h, v1.4h311; CHECK-NEXT: ret312 %tmp1 = load <4 x i16>, ptr %A313 %tmp2 = load <4 x i16>, ptr %B314 %tmp3 = zext <4 x i16> %tmp1 to <4 x i32>315 %tmp4 = zext <4 x i16> %tmp2 to <4 x i32>316 %tmp5 = mul <4 x i32> %tmp3, %tmp4317 ret <4 x i32> %tmp5318}319 320define <2 x i64> @umull_v2i32_v2i64(ptr %A, ptr %B) nounwind {321; CHECK-LABEL: umull_v2i32_v2i64:322; CHECK: // %bb.0:323; CHECK-NEXT: ldr d0, [x0]324; CHECK-NEXT: ldr d1, [x1]325; CHECK-NEXT: umull v0.2d, v0.2s, v1.2s326; CHECK-NEXT: ret327 %tmp1 = load <2 x i32>, ptr %A328 %tmp2 = load <2 x i32>, ptr %B329 %tmp3 = zext <2 x i32> %tmp1 to <2 x i64>330 %tmp4 = zext <2 x i32> %tmp2 to <2 x i64>331 %tmp5 = mul <2 x i64> %tmp3, %tmp4332 ret <2 x i64> %tmp5333}334 335define <8 x i16> @amull_v8i8_v8i16(ptr %A, ptr %B) nounwind {336; CHECK-NEON-LABEL: amull_v8i8_v8i16:337; CHECK-NEON: // %bb.0:338; CHECK-NEON-NEXT: ldr d0, [x0]339; CHECK-NEON-NEXT: ldr d1, [x1]340; CHECK-NEON-NEXT: smull v0.8h, v0.8b, v1.8b341; CHECK-NEON-NEXT: bic v0.8h, #255, lsl #8342; CHECK-NEON-NEXT: ret343;344; CHECK-SVE-LABEL: amull_v8i8_v8i16:345; CHECK-SVE: // %bb.0:346; CHECK-SVE-NEXT: ldr d0, [x0]347; CHECK-SVE-NEXT: ldr d1, [x1]348; CHECK-SVE-NEXT: smull v0.8h, v0.8b, v1.8b349; CHECK-SVE-NEXT: bic v0.8h, #255, lsl #8350; CHECK-SVE-NEXT: ret351;352; CHECK-GI-LABEL: amull_v8i8_v8i16:353; CHECK-GI: // %bb.0:354; CHECK-GI-NEXT: ldr d1, [x0]355; CHECK-GI-NEXT: ldr d2, [x1]356; CHECK-GI-NEXT: movi v0.2d, #0xff00ff00ff00ff357; CHECK-GI-NEXT: umull v1.8h, v1.8b, v2.8b358; CHECK-GI-NEXT: and v0.16b, v1.16b, v0.16b359; CHECK-GI-NEXT: ret360 %tmp1 = load <8 x i8>, ptr %A361 %tmp2 = load <8 x i8>, ptr %B362 %tmp3 = zext <8 x i8> %tmp1 to <8 x i16>363 %tmp4 = zext <8 x i8> %tmp2 to <8 x i16>364 %tmp5 = mul <8 x i16> %tmp3, %tmp4365 %and = and <8 x i16> %tmp5, <i16 255, i16 255, i16 255, i16 255, i16 255, i16 255, i16 255, i16 255>366 ret <8 x i16> %and367}368 369define <4 x i32> @amull_v4i16_v4i32(ptr %A, ptr %B) nounwind {370; CHECK-NEON-LABEL: amull_v4i16_v4i32:371; CHECK-NEON: // %bb.0:372; CHECK-NEON-NEXT: ldr d1, [x0]373; CHECK-NEON-NEXT: ldr d2, [x1]374; CHECK-NEON-NEXT: movi v0.2d, #0x00ffff0000ffff375; CHECK-NEON-NEXT: smull v1.4s, v1.4h, v2.4h376; CHECK-NEON-NEXT: and v0.16b, v1.16b, v0.16b377; CHECK-NEON-NEXT: ret378;379; CHECK-SVE-LABEL: amull_v4i16_v4i32:380; CHECK-SVE: // %bb.0:381; CHECK-SVE-NEXT: ldr d1, [x0]382; CHECK-SVE-NEXT: ldr d2, [x1]383; CHECK-SVE-NEXT: movi v0.2d, #0x00ffff0000ffff384; CHECK-SVE-NEXT: smull v1.4s, v1.4h, v2.4h385; CHECK-SVE-NEXT: and v0.16b, v1.16b, v0.16b386; CHECK-SVE-NEXT: ret387;388; CHECK-GI-LABEL: amull_v4i16_v4i32:389; CHECK-GI: // %bb.0:390; CHECK-GI-NEXT: ldr d1, [x0]391; CHECK-GI-NEXT: ldr d2, [x1]392; CHECK-GI-NEXT: movi v0.2d, #0x00ffff0000ffff393; CHECK-GI-NEXT: umull v1.4s, v1.4h, v2.4h394; CHECK-GI-NEXT: and v0.16b, v1.16b, v0.16b395; CHECK-GI-NEXT: ret396 %tmp1 = load <4 x i16>, ptr %A397 %tmp2 = load <4 x i16>, ptr %B398 %tmp3 = zext <4 x i16> %tmp1 to <4 x i32>399 %tmp4 = zext <4 x i16> %tmp2 to <4 x i32>400 %tmp5 = mul <4 x i32> %tmp3, %tmp4401 %and = and <4 x i32> %tmp5, <i32 65535, i32 65535, i32 65535, i32 65535>402 ret <4 x i32> %and403}404 405define <2 x i64> @amull_v2i32_v2i64(ptr %A, ptr %B) nounwind {406; CHECK-NEON-LABEL: amull_v2i32_v2i64:407; CHECK-NEON: // %bb.0:408; CHECK-NEON-NEXT: ldr d1, [x0]409; CHECK-NEON-NEXT: ldr d2, [x1]410; CHECK-NEON-NEXT: movi v0.2d, #0x000000ffffffff411; CHECK-NEON-NEXT: smull v1.2d, v1.2s, v2.2s412; CHECK-NEON-NEXT: and v0.16b, v1.16b, v0.16b413; CHECK-NEON-NEXT: ret414;415; CHECK-SVE-LABEL: amull_v2i32_v2i64:416; CHECK-SVE: // %bb.0:417; CHECK-SVE-NEXT: ldr d1, [x0]418; CHECK-SVE-NEXT: ldr d2, [x1]419; CHECK-SVE-NEXT: movi v0.2d, #0x000000ffffffff420; CHECK-SVE-NEXT: smull v1.2d, v1.2s, v2.2s421; CHECK-SVE-NEXT: and v0.16b, v1.16b, v0.16b422; CHECK-SVE-NEXT: ret423;424; CHECK-GI-LABEL: amull_v2i32_v2i64:425; CHECK-GI: // %bb.0:426; CHECK-GI-NEXT: ldr d1, [x0]427; CHECK-GI-NEXT: ldr d2, [x1]428; CHECK-GI-NEXT: movi v0.2d, #0x000000ffffffff429; CHECK-GI-NEXT: umull v1.2d, v1.2s, v2.2s430; CHECK-GI-NEXT: and v0.16b, v1.16b, v0.16b431; CHECK-GI-NEXT: ret432 %tmp1 = load <2 x i32>, ptr %A433 %tmp2 = load <2 x i32>, ptr %B434 %tmp3 = zext <2 x i32> %tmp1 to <2 x i64>435 %tmp4 = zext <2 x i32> %tmp2 to <2 x i64>436 %tmp5 = mul <2 x i64> %tmp3, %tmp4437 %and = and <2 x i64> %tmp5, <i64 4294967295, i64 4294967295>438 ret <2 x i64> %and439}440 441define <8 x i16> @smlal_v8i8_v8i16(ptr %A, ptr %B, ptr %C) nounwind {442; CHECK-LABEL: smlal_v8i8_v8i16:443; CHECK: // %bb.0:444; CHECK-NEXT: ldr q0, [x0]445; CHECK-NEXT: ldr d1, [x1]446; CHECK-NEXT: ldr d2, [x2]447; CHECK-NEXT: smlal v0.8h, v1.8b, v2.8b448; CHECK-NEXT: ret449 %tmp1 = load <8 x i16>, ptr %A450 %tmp2 = load <8 x i8>, ptr %B451 %tmp3 = load <8 x i8>, ptr %C452 %tmp4 = sext <8 x i8> %tmp2 to <8 x i16>453 %tmp5 = sext <8 x i8> %tmp3 to <8 x i16>454 %tmp6 = mul <8 x i16> %tmp4, %tmp5455 %tmp7 = add <8 x i16> %tmp1, %tmp6456 ret <8 x i16> %tmp7457}458 459define <4 x i32> @smlal_v4i16_v4i32(ptr %A, ptr %B, ptr %C) nounwind {460; CHECK-LABEL: smlal_v4i16_v4i32:461; CHECK: // %bb.0:462; CHECK-NEXT: ldr q0, [x0]463; CHECK-NEXT: ldr d1, [x1]464; CHECK-NEXT: ldr d2, [x2]465; CHECK-NEXT: smlal v0.4s, v1.4h, v2.4h466; CHECK-NEXT: ret467 %tmp1 = load <4 x i32>, ptr %A468 %tmp2 = load <4 x i16>, ptr %B469 %tmp3 = load <4 x i16>, ptr %C470 %tmp4 = sext <4 x i16> %tmp2 to <4 x i32>471 %tmp5 = sext <4 x i16> %tmp3 to <4 x i32>472 %tmp6 = mul <4 x i32> %tmp4, %tmp5473 %tmp7 = add <4 x i32> %tmp1, %tmp6474 ret <4 x i32> %tmp7475}476 477define <2 x i64> @smlal_v2i32_v2i64(ptr %A, ptr %B, ptr %C) nounwind {478; CHECK-LABEL: smlal_v2i32_v2i64:479; CHECK: // %bb.0:480; CHECK-NEXT: ldr q0, [x0]481; CHECK-NEXT: ldr d1, [x1]482; CHECK-NEXT: ldr d2, [x2]483; CHECK-NEXT: smlal v0.2d, v1.2s, v2.2s484; CHECK-NEXT: ret485 %tmp1 = load <2 x i64>, ptr %A486 %tmp2 = load <2 x i32>, ptr %B487 %tmp3 = load <2 x i32>, ptr %C488 %tmp4 = sext <2 x i32> %tmp2 to <2 x i64>489 %tmp5 = sext <2 x i32> %tmp3 to <2 x i64>490 %tmp6 = mul <2 x i64> %tmp4, %tmp5491 %tmp7 = add <2 x i64> %tmp1, %tmp6492 ret <2 x i64> %tmp7493}494 495define <8 x i16> @umlal_v8i8_v8i16(ptr %A, ptr %B, ptr %C) nounwind {496; CHECK-LABEL: umlal_v8i8_v8i16:497; CHECK: // %bb.0:498; CHECK-NEXT: ldr q0, [x0]499; CHECK-NEXT: ldr d1, [x1]500; CHECK-NEXT: ldr d2, [x2]501; CHECK-NEXT: umlal v0.8h, v1.8b, v2.8b502; CHECK-NEXT: ret503 %tmp1 = load <8 x i16>, ptr %A504 %tmp2 = load <8 x i8>, ptr %B505 %tmp3 = load <8 x i8>, ptr %C506 %tmp4 = zext <8 x i8> %tmp2 to <8 x i16>507 %tmp5 = zext <8 x i8> %tmp3 to <8 x i16>508 %tmp6 = mul <8 x i16> %tmp4, %tmp5509 %tmp7 = add <8 x i16> %tmp1, %tmp6510 ret <8 x i16> %tmp7511}512 513define <4 x i32> @umlal_v4i16_v4i32(ptr %A, ptr %B, ptr %C) nounwind {514; CHECK-LABEL: umlal_v4i16_v4i32:515; CHECK: // %bb.0:516; CHECK-NEXT: ldr q0, [x0]517; CHECK-NEXT: ldr d1, [x1]518; CHECK-NEXT: ldr d2, [x2]519; CHECK-NEXT: umlal v0.4s, v1.4h, v2.4h520; CHECK-NEXT: ret521 %tmp1 = load <4 x i32>, ptr %A522 %tmp2 = load <4 x i16>, ptr %B523 %tmp3 = load <4 x i16>, ptr %C524 %tmp4 = zext <4 x i16> %tmp2 to <4 x i32>525 %tmp5 = zext <4 x i16> %tmp3 to <4 x i32>526 %tmp6 = mul <4 x i32> %tmp4, %tmp5527 %tmp7 = add <4 x i32> %tmp1, %tmp6528 ret <4 x i32> %tmp7529}530 531define <2 x i64> @umlal_v2i32_v2i64(ptr %A, ptr %B, ptr %C) nounwind {532; CHECK-LABEL: umlal_v2i32_v2i64:533; CHECK: // %bb.0:534; CHECK-NEXT: ldr q0, [x0]535; CHECK-NEXT: ldr d1, [x1]536; CHECK-NEXT: ldr d2, [x2]537; CHECK-NEXT: umlal v0.2d, v1.2s, v2.2s538; CHECK-NEXT: ret539 %tmp1 = load <2 x i64>, ptr %A540 %tmp2 = load <2 x i32>, ptr %B541 %tmp3 = load <2 x i32>, ptr %C542 %tmp4 = zext <2 x i32> %tmp2 to <2 x i64>543 %tmp5 = zext <2 x i32> %tmp3 to <2 x i64>544 %tmp6 = mul <2 x i64> %tmp4, %tmp5545 %tmp7 = add <2 x i64> %tmp1, %tmp6546 ret <2 x i64> %tmp7547}548 549define <8 x i16> @amlal_v8i8_v8i16(ptr %A, ptr %B, ptr %C) nounwind {550; CHECK-NEON-LABEL: amlal_v8i8_v8i16:551; CHECK-NEON: // %bb.0:552; CHECK-NEON-NEXT: ldr q0, [x0]553; CHECK-NEON-NEXT: ldr d1, [x1]554; CHECK-NEON-NEXT: ldr d2, [x2]555; CHECK-NEON-NEXT: smlal v0.8h, v1.8b, v2.8b556; CHECK-NEON-NEXT: bic v0.8h, #255, lsl #8557; CHECK-NEON-NEXT: ret558;559; CHECK-SVE-LABEL: amlal_v8i8_v8i16:560; CHECK-SVE: // %bb.0:561; CHECK-SVE-NEXT: ldr q0, [x0]562; CHECK-SVE-NEXT: ldr d1, [x1]563; CHECK-SVE-NEXT: ldr d2, [x2]564; CHECK-SVE-NEXT: smlal v0.8h, v1.8b, v2.8b565; CHECK-SVE-NEXT: bic v0.8h, #255, lsl #8566; CHECK-SVE-NEXT: ret567;568; CHECK-GI-LABEL: amlal_v8i8_v8i16:569; CHECK-GI: // %bb.0:570; CHECK-GI-NEXT: ldr q0, [x0]571; CHECK-GI-NEXT: ldr d1, [x1]572; CHECK-GI-NEXT: movi v3.2d, #0xff00ff00ff00ff573; CHECK-GI-NEXT: ldr d2, [x2]574; CHECK-GI-NEXT: umlal v0.8h, v1.8b, v2.8b575; CHECK-GI-NEXT: and v0.16b, v0.16b, v3.16b576; CHECK-GI-NEXT: ret577 %tmp1 = load <8 x i16>, ptr %A578 %tmp2 = load <8 x i8>, ptr %B579 %tmp3 = load <8 x i8>, ptr %C580 %tmp4 = zext <8 x i8> %tmp2 to <8 x i16>581 %tmp5 = zext <8 x i8> %tmp3 to <8 x i16>582 %tmp6 = mul <8 x i16> %tmp4, %tmp5583 %tmp7 = add <8 x i16> %tmp1, %tmp6584 %and = and <8 x i16> %tmp7, <i16 255, i16 255, i16 255, i16 255, i16 255, i16 255, i16 255, i16 255>585 ret <8 x i16> %and586}587 588define <4 x i32> @amlal_v4i16_v4i32(ptr %A, ptr %B, ptr %C) nounwind {589; CHECK-NEON-LABEL: amlal_v4i16_v4i32:590; CHECK-NEON: // %bb.0:591; CHECK-NEON-NEXT: ldr q0, [x0]592; CHECK-NEON-NEXT: ldr d1, [x1]593; CHECK-NEON-NEXT: ldr d2, [x2]594; CHECK-NEON-NEXT: smlal v0.4s, v1.4h, v2.4h595; CHECK-NEON-NEXT: movi v1.2d, #0x00ffff0000ffff596; CHECK-NEON-NEXT: and v0.16b, v0.16b, v1.16b597; CHECK-NEON-NEXT: ret598;599; CHECK-SVE-LABEL: amlal_v4i16_v4i32:600; CHECK-SVE: // %bb.0:601; CHECK-SVE-NEXT: ldr q0, [x0]602; CHECK-SVE-NEXT: ldr d1, [x1]603; CHECK-SVE-NEXT: ldr d2, [x2]604; CHECK-SVE-NEXT: smlal v0.4s, v1.4h, v2.4h605; CHECK-SVE-NEXT: movi v1.2d, #0x00ffff0000ffff606; CHECK-SVE-NEXT: and v0.16b, v0.16b, v1.16b607; CHECK-SVE-NEXT: ret608;609; CHECK-GI-LABEL: amlal_v4i16_v4i32:610; CHECK-GI: // %bb.0:611; CHECK-GI-NEXT: ldr q0, [x0]612; CHECK-GI-NEXT: ldr d1, [x1]613; CHECK-GI-NEXT: movi v3.2d, #0x00ffff0000ffff614; CHECK-GI-NEXT: ldr d2, [x2]615; CHECK-GI-NEXT: umlal v0.4s, v1.4h, v2.4h616; CHECK-GI-NEXT: and v0.16b, v0.16b, v3.16b617; CHECK-GI-NEXT: ret618 %tmp1 = load <4 x i32>, ptr %A619 %tmp2 = load <4 x i16>, ptr %B620 %tmp3 = load <4 x i16>, ptr %C621 %tmp4 = zext <4 x i16> %tmp2 to <4 x i32>622 %tmp5 = zext <4 x i16> %tmp3 to <4 x i32>623 %tmp6 = mul <4 x i32> %tmp4, %tmp5624 %tmp7 = add <4 x i32> %tmp1, %tmp6625 %and = and <4 x i32> %tmp7, <i32 65535, i32 65535, i32 65535, i32 65535>626 ret <4 x i32> %and627}628 629define <2 x i64> @amlal_v2i32_v2i64(ptr %A, ptr %B, ptr %C) nounwind {630; CHECK-NEON-LABEL: amlal_v2i32_v2i64:631; CHECK-NEON: // %bb.0:632; CHECK-NEON-NEXT: ldr q0, [x0]633; CHECK-NEON-NEXT: ldr d1, [x1]634; CHECK-NEON-NEXT: ldr d2, [x2]635; CHECK-NEON-NEXT: smlal v0.2d, v1.2s, v2.2s636; CHECK-NEON-NEXT: movi v1.2d, #0x000000ffffffff637; CHECK-NEON-NEXT: and v0.16b, v0.16b, v1.16b638; CHECK-NEON-NEXT: ret639;640; CHECK-SVE-LABEL: amlal_v2i32_v2i64:641; CHECK-SVE: // %bb.0:642; CHECK-SVE-NEXT: ldr q0, [x0]643; CHECK-SVE-NEXT: ldr d1, [x1]644; CHECK-SVE-NEXT: ldr d2, [x2]645; CHECK-SVE-NEXT: smlal v0.2d, v1.2s, v2.2s646; CHECK-SVE-NEXT: movi v1.2d, #0x000000ffffffff647; CHECK-SVE-NEXT: and v0.16b, v0.16b, v1.16b648; CHECK-SVE-NEXT: ret649;650; CHECK-GI-LABEL: amlal_v2i32_v2i64:651; CHECK-GI: // %bb.0:652; CHECK-GI-NEXT: ldr q0, [x0]653; CHECK-GI-NEXT: ldr d1, [x1]654; CHECK-GI-NEXT: movi v3.2d, #0x000000ffffffff655; CHECK-GI-NEXT: ldr d2, [x2]656; CHECK-GI-NEXT: umlal v0.2d, v1.2s, v2.2s657; CHECK-GI-NEXT: and v0.16b, v0.16b, v3.16b658; CHECK-GI-NEXT: ret659 %tmp1 = load <2 x i64>, ptr %A660 %tmp2 = load <2 x i32>, ptr %B661 %tmp3 = load <2 x i32>, ptr %C662 %tmp4 = zext <2 x i32> %tmp2 to <2 x i64>663 %tmp5 = zext <2 x i32> %tmp3 to <2 x i64>664 %tmp6 = mul <2 x i64> %tmp4, %tmp5665 %tmp7 = add <2 x i64> %tmp1, %tmp6666 %and = and <2 x i64> %tmp7, <i64 4294967295, i64 4294967295>667 ret <2 x i64> %and668}669 670define <8 x i16> @smlsl_v8i8_v8i16(ptr %A, ptr %B, ptr %C) nounwind {671; CHECK-LABEL: smlsl_v8i8_v8i16:672; CHECK: // %bb.0:673; CHECK-NEXT: ldr q0, [x0]674; CHECK-NEXT: ldr d1, [x1]675; CHECK-NEXT: ldr d2, [x2]676; CHECK-NEXT: smlsl v0.8h, v1.8b, v2.8b677; CHECK-NEXT: ret678 %tmp1 = load <8 x i16>, ptr %A679 %tmp2 = load <8 x i8>, ptr %B680 %tmp3 = load <8 x i8>, ptr %C681 %tmp4 = sext <8 x i8> %tmp2 to <8 x i16>682 %tmp5 = sext <8 x i8> %tmp3 to <8 x i16>683 %tmp6 = mul <8 x i16> %tmp4, %tmp5684 %tmp7 = sub <8 x i16> %tmp1, %tmp6685 ret <8 x i16> %tmp7686}687 688define <4 x i32> @smlsl_v4i16_v4i32(ptr %A, ptr %B, ptr %C) nounwind {689; CHECK-LABEL: smlsl_v4i16_v4i32:690; CHECK: // %bb.0:691; CHECK-NEXT: ldr q0, [x0]692; CHECK-NEXT: ldr d1, [x1]693; CHECK-NEXT: ldr d2, [x2]694; CHECK-NEXT: smlsl v0.4s, v1.4h, v2.4h695; CHECK-NEXT: ret696 %tmp1 = load <4 x i32>, ptr %A697 %tmp2 = load <4 x i16>, ptr %B698 %tmp3 = load <4 x i16>, ptr %C699 %tmp4 = sext <4 x i16> %tmp2 to <4 x i32>700 %tmp5 = sext <4 x i16> %tmp3 to <4 x i32>701 %tmp6 = mul <4 x i32> %tmp4, %tmp5702 %tmp7 = sub <4 x i32> %tmp1, %tmp6703 ret <4 x i32> %tmp7704}705 706define <2 x i64> @smlsl_v2i32_v2i64(ptr %A, ptr %B, ptr %C) nounwind {707; CHECK-LABEL: smlsl_v2i32_v2i64:708; CHECK: // %bb.0:709; CHECK-NEXT: ldr q0, [x0]710; CHECK-NEXT: ldr d1, [x1]711; CHECK-NEXT: ldr d2, [x2]712; CHECK-NEXT: smlsl v0.2d, v1.2s, v2.2s713; CHECK-NEXT: ret714 %tmp1 = load <2 x i64>, ptr %A715 %tmp2 = load <2 x i32>, ptr %B716 %tmp3 = load <2 x i32>, ptr %C717 %tmp4 = sext <2 x i32> %tmp2 to <2 x i64>718 %tmp5 = sext <2 x i32> %tmp3 to <2 x i64>719 %tmp6 = mul <2 x i64> %tmp4, %tmp5720 %tmp7 = sub <2 x i64> %tmp1, %tmp6721 ret <2 x i64> %tmp7722}723 724define <8 x i16> @umlsl_v8i8_v8i16(ptr %A, ptr %B, ptr %C) nounwind {725; CHECK-LABEL: umlsl_v8i8_v8i16:726; CHECK: // %bb.0:727; CHECK-NEXT: ldr q0, [x0]728; CHECK-NEXT: ldr d1, [x1]729; CHECK-NEXT: ldr d2, [x2]730; CHECK-NEXT: umlsl v0.8h, v1.8b, v2.8b731; CHECK-NEXT: ret732 %tmp1 = load <8 x i16>, ptr %A733 %tmp2 = load <8 x i8>, ptr %B734 %tmp3 = load <8 x i8>, ptr %C735 %tmp4 = zext <8 x i8> %tmp2 to <8 x i16>736 %tmp5 = zext <8 x i8> %tmp3 to <8 x i16>737 %tmp6 = mul <8 x i16> %tmp4, %tmp5738 %tmp7 = sub <8 x i16> %tmp1, %tmp6739 ret <8 x i16> %tmp7740}741 742define <4 x i32> @umlsl_v4i16_v4i32(ptr %A, ptr %B, ptr %C) nounwind {743; CHECK-LABEL: umlsl_v4i16_v4i32:744; CHECK: // %bb.0:745; CHECK-NEXT: ldr q0, [x0]746; CHECK-NEXT: ldr d1, [x1]747; CHECK-NEXT: ldr d2, [x2]748; CHECK-NEXT: umlsl v0.4s, v1.4h, v2.4h749; CHECK-NEXT: ret750 %tmp1 = load <4 x i32>, ptr %A751 %tmp2 = load <4 x i16>, ptr %B752 %tmp3 = load <4 x i16>, ptr %C753 %tmp4 = zext <4 x i16> %tmp2 to <4 x i32>754 %tmp5 = zext <4 x i16> %tmp3 to <4 x i32>755 %tmp6 = mul <4 x i32> %tmp4, %tmp5756 %tmp7 = sub <4 x i32> %tmp1, %tmp6757 ret <4 x i32> %tmp7758}759 760define <2 x i64> @umlsl_v2i32_v2i64(ptr %A, ptr %B, ptr %C) nounwind {761; CHECK-LABEL: umlsl_v2i32_v2i64:762; CHECK: // %bb.0:763; CHECK-NEXT: ldr q0, [x0]764; CHECK-NEXT: ldr d1, [x1]765; CHECK-NEXT: ldr d2, [x2]766; CHECK-NEXT: umlsl v0.2d, v1.2s, v2.2s767; CHECK-NEXT: ret768 %tmp1 = load <2 x i64>, ptr %A769 %tmp2 = load <2 x i32>, ptr %B770 %tmp3 = load <2 x i32>, ptr %C771 %tmp4 = zext <2 x i32> %tmp2 to <2 x i64>772 %tmp5 = zext <2 x i32> %tmp3 to <2 x i64>773 %tmp6 = mul <2 x i64> %tmp4, %tmp5774 %tmp7 = sub <2 x i64> %tmp1, %tmp6775 ret <2 x i64> %tmp7776}777 778define <8 x i16> @amlsl_v8i8_v8i16(ptr %A, ptr %B, ptr %C) nounwind {779; CHECK-NEON-LABEL: amlsl_v8i8_v8i16:780; CHECK-NEON: // %bb.0:781; CHECK-NEON-NEXT: ldr q0, [x0]782; CHECK-NEON-NEXT: ldr d1, [x1]783; CHECK-NEON-NEXT: ldr d2, [x2]784; CHECK-NEON-NEXT: smlsl v0.8h, v1.8b, v2.8b785; CHECK-NEON-NEXT: bic v0.8h, #255, lsl #8786; CHECK-NEON-NEXT: ret787;788; CHECK-SVE-LABEL: amlsl_v8i8_v8i16:789; CHECK-SVE: // %bb.0:790; CHECK-SVE-NEXT: ldr q0, [x0]791; CHECK-SVE-NEXT: ldr d1, [x1]792; CHECK-SVE-NEXT: ldr d2, [x2]793; CHECK-SVE-NEXT: smlsl v0.8h, v1.8b, v2.8b794; CHECK-SVE-NEXT: bic v0.8h, #255, lsl #8795; CHECK-SVE-NEXT: ret796;797; CHECK-GI-LABEL: amlsl_v8i8_v8i16:798; CHECK-GI: // %bb.0:799; CHECK-GI-NEXT: ldr q0, [x0]800; CHECK-GI-NEXT: ldr d1, [x1]801; CHECK-GI-NEXT: movi v3.2d, #0xff00ff00ff00ff802; CHECK-GI-NEXT: ldr d2, [x2]803; CHECK-GI-NEXT: umlsl v0.8h, v1.8b, v2.8b804; CHECK-GI-NEXT: and v0.16b, v0.16b, v3.16b805; CHECK-GI-NEXT: ret806 %tmp1 = load <8 x i16>, ptr %A807 %tmp2 = load <8 x i8>, ptr %B808 %tmp3 = load <8 x i8>, ptr %C809 %tmp4 = zext <8 x i8> %tmp2 to <8 x i16>810 %tmp5 = zext <8 x i8> %tmp3 to <8 x i16>811 %tmp6 = mul <8 x i16> %tmp4, %tmp5812 %tmp7 = sub <8 x i16> %tmp1, %tmp6813 %and = and <8 x i16> %tmp7, <i16 255, i16 255, i16 255, i16 255, i16 255, i16 255, i16 255, i16 255>814 ret <8 x i16> %and815}816 817define <4 x i32> @amlsl_v4i16_v4i32(ptr %A, ptr %B, ptr %C) nounwind {818; CHECK-NEON-LABEL: amlsl_v4i16_v4i32:819; CHECK-NEON: // %bb.0:820; CHECK-NEON-NEXT: ldr q0, [x0]821; CHECK-NEON-NEXT: ldr d1, [x1]822; CHECK-NEON-NEXT: ldr d2, [x2]823; CHECK-NEON-NEXT: smlsl v0.4s, v1.4h, v2.4h824; CHECK-NEON-NEXT: movi v1.2d, #0x00ffff0000ffff825; CHECK-NEON-NEXT: and v0.16b, v0.16b, v1.16b826; CHECK-NEON-NEXT: ret827;828; CHECK-SVE-LABEL: amlsl_v4i16_v4i32:829; CHECK-SVE: // %bb.0:830; CHECK-SVE-NEXT: ldr q0, [x0]831; CHECK-SVE-NEXT: ldr d1, [x1]832; CHECK-SVE-NEXT: ldr d2, [x2]833; CHECK-SVE-NEXT: smlsl v0.4s, v1.4h, v2.4h834; CHECK-SVE-NEXT: movi v1.2d, #0x00ffff0000ffff835; CHECK-SVE-NEXT: and v0.16b, v0.16b, v1.16b836; CHECK-SVE-NEXT: ret837;838; CHECK-GI-LABEL: amlsl_v4i16_v4i32:839; CHECK-GI: // %bb.0:840; CHECK-GI-NEXT: ldr q0, [x0]841; CHECK-GI-NEXT: ldr d1, [x1]842; CHECK-GI-NEXT: movi v3.2d, #0x00ffff0000ffff843; CHECK-GI-NEXT: ldr d2, [x2]844; CHECK-GI-NEXT: umlsl v0.4s, v1.4h, v2.4h845; CHECK-GI-NEXT: and v0.16b, v0.16b, v3.16b846; CHECK-GI-NEXT: ret847 %tmp1 = load <4 x i32>, ptr %A848 %tmp2 = load <4 x i16>, ptr %B849 %tmp3 = load <4 x i16>, ptr %C850 %tmp4 = zext <4 x i16> %tmp2 to <4 x i32>851 %tmp5 = zext <4 x i16> %tmp3 to <4 x i32>852 %tmp6 = mul <4 x i32> %tmp4, %tmp5853 %tmp7 = sub <4 x i32> %tmp1, %tmp6854 %and = and <4 x i32> %tmp7, <i32 65535, i32 65535, i32 65535, i32 65535>855 ret <4 x i32> %and856}857 858define <2 x i64> @amlsl_v2i32_v2i64(ptr %A, ptr %B, ptr %C) nounwind {859; CHECK-NEON-LABEL: amlsl_v2i32_v2i64:860; CHECK-NEON: // %bb.0:861; CHECK-NEON-NEXT: ldr q0, [x0]862; CHECK-NEON-NEXT: ldr d1, [x1]863; CHECK-NEON-NEXT: ldr d2, [x2]864; CHECK-NEON-NEXT: smlsl v0.2d, v1.2s, v2.2s865; CHECK-NEON-NEXT: movi v1.2d, #0x000000ffffffff866; CHECK-NEON-NEXT: and v0.16b, v0.16b, v1.16b867; CHECK-NEON-NEXT: ret868;869; CHECK-SVE-LABEL: amlsl_v2i32_v2i64:870; CHECK-SVE: // %bb.0:871; CHECK-SVE-NEXT: ldr q0, [x0]872; CHECK-SVE-NEXT: ldr d1, [x1]873; CHECK-SVE-NEXT: ldr d2, [x2]874; CHECK-SVE-NEXT: smlsl v0.2d, v1.2s, v2.2s875; CHECK-SVE-NEXT: movi v1.2d, #0x000000ffffffff876; CHECK-SVE-NEXT: and v0.16b, v0.16b, v1.16b877; CHECK-SVE-NEXT: ret878;879; CHECK-GI-LABEL: amlsl_v2i32_v2i64:880; CHECK-GI: // %bb.0:881; CHECK-GI-NEXT: ldr q0, [x0]882; CHECK-GI-NEXT: ldr d1, [x1]883; CHECK-GI-NEXT: movi v3.2d, #0x000000ffffffff884; CHECK-GI-NEXT: ldr d2, [x2]885; CHECK-GI-NEXT: umlsl v0.2d, v1.2s, v2.2s886; CHECK-GI-NEXT: and v0.16b, v0.16b, v3.16b887; CHECK-GI-NEXT: ret888 %tmp1 = load <2 x i64>, ptr %A889 %tmp2 = load <2 x i32>, ptr %B890 %tmp3 = load <2 x i32>, ptr %C891 %tmp4 = zext <2 x i32> %tmp2 to <2 x i64>892 %tmp5 = zext <2 x i32> %tmp3 to <2 x i64>893 %tmp6 = mul <2 x i64> %tmp4, %tmp5894 %tmp7 = sub <2 x i64> %tmp1, %tmp6895 %and = and <2 x i64> %tmp7, <i64 4294967295, i64 4294967295>896 ret <2 x i64> %and897}898 899; SMULL recognizing BUILD_VECTORs with sign/zero-extended elements.900define <8 x i16> @smull_extvec_v8i8_v8i16(<8 x i8> %arg) nounwind {901; CHECK-LABEL: smull_extvec_v8i8_v8i16:902; CHECK: // %bb.0:903; CHECK-NEXT: movi v1.8b, #244904; CHECK-NEXT: smull v0.8h, v0.8b, v1.8b905; CHECK-NEXT: ret906 %tmp3 = sext <8 x i8> %arg to <8 x i16>907 %tmp4 = mul <8 x i16> %tmp3, <i16 -12, i16 -12, i16 -12, i16 -12, i16 -12, i16 -12, i16 -12, i16 -12>908 ret <8 x i16> %tmp4909}910 911define <8 x i16> @smull_noextvec_v8i8_v8i16(<8 x i8> %arg) nounwind {912; Do not use SMULL if the BUILD_VECTOR element values are too big.913; CHECK-NEON-LABEL: smull_noextvec_v8i8_v8i16:914; CHECK-NEON: // %bb.0:915; CHECK-NEON-NEXT: mov w8, #64537 // =0xfc19916; CHECK-NEON-NEXT: sshll v0.8h, v0.8b, #0917; CHECK-NEON-NEXT: dup v1.8h, w8918; CHECK-NEON-NEXT: mul v0.8h, v0.8h, v1.8h919; CHECK-NEON-NEXT: ret920;921; CHECK-SVE-LABEL: smull_noextvec_v8i8_v8i16:922; CHECK-SVE: // %bb.0:923; CHECK-SVE-NEXT: mov w8, #64537 // =0xfc19924; CHECK-SVE-NEXT: sshll v0.8h, v0.8b, #0925; CHECK-SVE-NEXT: dup v1.8h, w8926; CHECK-SVE-NEXT: mul v0.8h, v0.8h, v1.8h927; CHECK-SVE-NEXT: ret928;929; CHECK-GI-LABEL: smull_noextvec_v8i8_v8i16:930; CHECK-GI: // %bb.0:931; CHECK-GI-NEXT: adrp x8, .LCPI34_0932; CHECK-GI-NEXT: sshll v0.8h, v0.8b, #0933; CHECK-GI-NEXT: ldr q1, [x8, :lo12:.LCPI34_0]934; CHECK-GI-NEXT: mul v0.8h, v0.8h, v1.8h935; CHECK-GI-NEXT: ret936 %tmp3 = sext <8 x i8> %arg to <8 x i16>937 %tmp4 = mul <8 x i16> %tmp3, <i16 -999, i16 -999, i16 -999, i16 -999, i16 -999, i16 -999, i16 -999, i16 -999>938 ret <8 x i16> %tmp4939}940 941define <4 x i32> @smull_extvec_v4i16_v4i32(<4 x i16> %arg) nounwind {942; CHECK-LABEL: smull_extvec_v4i16_v4i32:943; CHECK: // %bb.0:944; CHECK-NEXT: mvni v1.4h, #11945; CHECK-NEXT: smull v0.4s, v0.4h, v1.4h946; CHECK-NEXT: ret947 %tmp3 = sext <4 x i16> %arg to <4 x i32>948 %tmp4 = mul <4 x i32> %tmp3, <i32 -12, i32 -12, i32 -12, i32 -12>949 ret <4 x i32> %tmp4950}951 952define <2 x i64> @smull_extvec_v2i32_v2i64(<2 x i32> %arg) nounwind {953; CHECK-NEON-LABEL: smull_extvec_v2i32_v2i64:954; CHECK-NEON: // %bb.0:955; CHECK-NEON-NEXT: mov w8, #-1234 // =0xfffffb2e956; CHECK-NEON-NEXT: dup v1.2s, w8957; CHECK-NEON-NEXT: smull v0.2d, v0.2s, v1.2s958; CHECK-NEON-NEXT: ret959;960; CHECK-SVE-LABEL: smull_extvec_v2i32_v2i64:961; CHECK-SVE: // %bb.0:962; CHECK-SVE-NEXT: mov w8, #-1234 // =0xfffffb2e963; CHECK-SVE-NEXT: dup v1.2s, w8964; CHECK-SVE-NEXT: smull v0.2d, v0.2s, v1.2s965; CHECK-SVE-NEXT: ret966;967; CHECK-GI-LABEL: smull_extvec_v2i32_v2i64:968; CHECK-GI: // %bb.0:969; CHECK-GI-NEXT: adrp x8, .LCPI36_0970; CHECK-GI-NEXT: ldr d1, [x8, :lo12:.LCPI36_0]971; CHECK-GI-NEXT: smull v0.2d, v0.2s, v1.2s972; CHECK-GI-NEXT: ret973 %tmp3 = sext <2 x i32> %arg to <2 x i64>974 %tmp4 = mul <2 x i64> %tmp3, <i64 -1234, i64 -1234>975 ret <2 x i64> %tmp4976}977 978define <8 x i16> @umull_extvec_v8i8_v8i16(<8 x i8> %arg) nounwind {979; CHECK-LABEL: umull_extvec_v8i8_v8i16:980; CHECK: // %bb.0:981; CHECK-NEXT: movi v1.8b, #12982; CHECK-NEXT: umull v0.8h, v0.8b, v1.8b983; CHECK-NEXT: ret984 %tmp3 = zext <8 x i8> %arg to <8 x i16>985 %tmp4 = mul <8 x i16> %tmp3, <i16 12, i16 12, i16 12, i16 12, i16 12, i16 12, i16 12, i16 12>986 ret <8 x i16> %tmp4987}988 989define <8 x i16> @umull_noextvec_v8i8_v8i16(<8 x i8> %arg) nounwind {990; Do not use SMULL if the BUILD_VECTOR element values are too big.991; CHECK-NEON-LABEL: umull_noextvec_v8i8_v8i16:992; CHECK-NEON: // %bb.0:993; CHECK-NEON-NEXT: mov w8, #999 // =0x3e7994; CHECK-NEON-NEXT: ushll v0.8h, v0.8b, #0995; CHECK-NEON-NEXT: dup v1.8h, w8996; CHECK-NEON-NEXT: mul v0.8h, v0.8h, v1.8h997; CHECK-NEON-NEXT: ret998;999; CHECK-SVE-LABEL: umull_noextvec_v8i8_v8i16:1000; CHECK-SVE: // %bb.0:1001; CHECK-SVE-NEXT: mov w8, #999 // =0x3e71002; CHECK-SVE-NEXT: ushll v0.8h, v0.8b, #01003; CHECK-SVE-NEXT: dup v1.8h, w81004; CHECK-SVE-NEXT: mul v0.8h, v0.8h, v1.8h1005; CHECK-SVE-NEXT: ret1006;1007; CHECK-GI-LABEL: umull_noextvec_v8i8_v8i16:1008; CHECK-GI: // %bb.0:1009; CHECK-GI-NEXT: adrp x8, .LCPI38_01010; CHECK-GI-NEXT: ushll v0.8h, v0.8b, #01011; CHECK-GI-NEXT: ldr q1, [x8, :lo12:.LCPI38_0]1012; CHECK-GI-NEXT: mul v0.8h, v0.8h, v1.8h1013; CHECK-GI-NEXT: ret1014 %tmp3 = zext <8 x i8> %arg to <8 x i16>1015 %tmp4 = mul <8 x i16> %tmp3, <i16 999, i16 999, i16 999, i16 999, i16 999, i16 999, i16 999, i16 999>1016 ret <8 x i16> %tmp41017}1018 1019define <4 x i32> @umull_extvec_v4i16_v4i32(<4 x i16> %arg) nounwind {1020; CHECK-NEON-LABEL: umull_extvec_v4i16_v4i32:1021; CHECK-NEON: // %bb.0:1022; CHECK-NEON-NEXT: mov w8, #1234 // =0x4d21023; CHECK-NEON-NEXT: dup v1.4h, w81024; CHECK-NEON-NEXT: umull v0.4s, v0.4h, v1.4h1025; CHECK-NEON-NEXT: ret1026;1027; CHECK-SVE-LABEL: umull_extvec_v4i16_v4i32:1028; CHECK-SVE: // %bb.0:1029; CHECK-SVE-NEXT: mov w8, #1234 // =0x4d21030; CHECK-SVE-NEXT: dup v1.4h, w81031; CHECK-SVE-NEXT: umull v0.4s, v0.4h, v1.4h1032; CHECK-SVE-NEXT: ret1033;1034; CHECK-GI-LABEL: umull_extvec_v4i16_v4i32:1035; CHECK-GI: // %bb.0:1036; CHECK-GI-NEXT: adrp x8, .LCPI39_01037; CHECK-GI-NEXT: ldr d1, [x8, :lo12:.LCPI39_0]1038; CHECK-GI-NEXT: umull v0.4s, v0.4h, v1.4h1039; CHECK-GI-NEXT: ret1040 %tmp3 = zext <4 x i16> %arg to <4 x i32>1041 %tmp4 = mul <4 x i32> %tmp3, <i32 1234, i32 1234, i32 1234, i32 1234>1042 ret <4 x i32> %tmp41043}1044 1045define <2 x i64> @umull_extvec_v2i32_v2i64(<2 x i32> %arg) nounwind {1046; CHECK-NEON-LABEL: umull_extvec_v2i32_v2i64:1047; CHECK-NEON: // %bb.0:1048; CHECK-NEON-NEXT: mov w8, #1234 // =0x4d21049; CHECK-NEON-NEXT: dup v1.2s, w81050; CHECK-NEON-NEXT: umull v0.2d, v0.2s, v1.2s1051; CHECK-NEON-NEXT: ret1052;1053; CHECK-SVE-LABEL: umull_extvec_v2i32_v2i64:1054; CHECK-SVE: // %bb.0:1055; CHECK-SVE-NEXT: mov w8, #1234 // =0x4d21056; CHECK-SVE-NEXT: dup v1.2s, w81057; CHECK-SVE-NEXT: umull v0.2d, v0.2s, v1.2s1058; CHECK-SVE-NEXT: ret1059;1060; CHECK-GI-LABEL: umull_extvec_v2i32_v2i64:1061; CHECK-GI: // %bb.0:1062; CHECK-GI-NEXT: adrp x8, .LCPI40_01063; CHECK-GI-NEXT: ldr d1, [x8, :lo12:.LCPI40_0]1064; CHECK-GI-NEXT: umull v0.2d, v0.2s, v1.2s1065; CHECK-GI-NEXT: ret1066 %tmp3 = zext <2 x i32> %arg to <2 x i64>1067 %tmp4 = mul <2 x i64> %tmp3, <i64 1234, i64 1234>1068 ret <2 x i64> %tmp41069}1070 1071define <8 x i16> @amull_extvec_v8i8_v8i16(<8 x i8> %arg) nounwind {1072; CHECK-NEON-LABEL: amull_extvec_v8i8_v8i16:1073; CHECK-NEON: // %bb.0:1074; CHECK-NEON-NEXT: movi v1.8b, #121075; CHECK-NEON-NEXT: smull v0.8h, v0.8b, v1.8b1076; CHECK-NEON-NEXT: bic v0.8h, #255, lsl #81077; CHECK-NEON-NEXT: ret1078;1079; CHECK-SVE-LABEL: amull_extvec_v8i8_v8i16:1080; CHECK-SVE: // %bb.0:1081; CHECK-SVE-NEXT: movi v1.8b, #121082; CHECK-SVE-NEXT: smull v0.8h, v0.8b, v1.8b1083; CHECK-SVE-NEXT: bic v0.8h, #255, lsl #81084; CHECK-SVE-NEXT: ret1085;1086; CHECK-GI-LABEL: amull_extvec_v8i8_v8i16:1087; CHECK-GI: // %bb.0:1088; CHECK-GI-NEXT: movi v1.8b, #121089; CHECK-GI-NEXT: movi v2.2d, #0xff00ff00ff00ff1090; CHECK-GI-NEXT: umull v0.8h, v0.8b, v1.8b1091; CHECK-GI-NEXT: and v0.16b, v0.16b, v2.16b1092; CHECK-GI-NEXT: ret1093 %tmp3 = zext <8 x i8> %arg to <8 x i16>1094 %tmp4 = mul <8 x i16> %tmp3, <i16 12, i16 12, i16 12, i16 12, i16 12, i16 12, i16 12, i16 12>1095 %and = and <8 x i16> %tmp4, <i16 255, i16 255, i16 255, i16 255, i16 255, i16 255, i16 255, i16 255>1096 ret <8 x i16> %and1097}1098 1099define <4 x i32> @amull_extvec_v4i16_v4i32(<4 x i16> %arg) nounwind {1100; CHECK-NEON-LABEL: amull_extvec_v4i16_v4i32:1101; CHECK-NEON: // %bb.0:1102; CHECK-NEON-NEXT: mov w8, #1234 // =0x4d21103; CHECK-NEON-NEXT: dup v1.4h, w81104; CHECK-NEON-NEXT: smull v0.4s, v0.4h, v1.4h1105; CHECK-NEON-NEXT: movi v1.2d, #0x00ffff0000ffff1106; CHECK-NEON-NEXT: and v0.16b, v0.16b, v1.16b1107; CHECK-NEON-NEXT: ret1108;1109; CHECK-SVE-LABEL: amull_extvec_v4i16_v4i32:1110; CHECK-SVE: // %bb.0:1111; CHECK-SVE-NEXT: mov w8, #1234 // =0x4d21112; CHECK-SVE-NEXT: dup v1.4h, w81113; CHECK-SVE-NEXT: smull v0.4s, v0.4h, v1.4h1114; CHECK-SVE-NEXT: movi v1.2d, #0x00ffff0000ffff1115; CHECK-SVE-NEXT: and v0.16b, v0.16b, v1.16b1116; CHECK-SVE-NEXT: ret1117;1118; CHECK-GI-LABEL: amull_extvec_v4i16_v4i32:1119; CHECK-GI: // %bb.0:1120; CHECK-GI-NEXT: adrp x8, .LCPI42_01121; CHECK-GI-NEXT: movi v2.2d, #0x00ffff0000ffff1122; CHECK-GI-NEXT: ldr d1, [x8, :lo12:.LCPI42_0]1123; CHECK-GI-NEXT: umull v0.4s, v0.4h, v1.4h1124; CHECK-GI-NEXT: and v0.16b, v0.16b, v2.16b1125; CHECK-GI-NEXT: ret1126 %tmp3 = zext <4 x i16> %arg to <4 x i32>1127 %tmp4 = mul <4 x i32> %tmp3, <i32 1234, i32 1234, i32 1234, i32 1234>1128 %and = and <4 x i32> %tmp4, <i32 65535, i32 65535, i32 65535, i32 65535>1129 ret <4 x i32> %and1130}1131 1132define <2 x i64> @amull_extvec_v2i32_v2i64(<2 x i32> %arg) nounwind {1133; CHECK-NEON-LABEL: amull_extvec_v2i32_v2i64:1134; CHECK-NEON: // %bb.0:1135; CHECK-NEON-NEXT: mov w8, #1234 // =0x4d21136; CHECK-NEON-NEXT: dup v1.2s, w81137; CHECK-NEON-NEXT: smull v0.2d, v0.2s, v1.2s1138; CHECK-NEON-NEXT: movi v1.2d, #0x000000ffffffff1139; CHECK-NEON-NEXT: and v0.16b, v0.16b, v1.16b1140; CHECK-NEON-NEXT: ret1141;1142; CHECK-SVE-LABEL: amull_extvec_v2i32_v2i64:1143; CHECK-SVE: // %bb.0:1144; CHECK-SVE-NEXT: mov w8, #1234 // =0x4d21145; CHECK-SVE-NEXT: dup v1.2s, w81146; CHECK-SVE-NEXT: smull v0.2d, v0.2s, v1.2s1147; CHECK-SVE-NEXT: movi v1.2d, #0x000000ffffffff1148; CHECK-SVE-NEXT: and v0.16b, v0.16b, v1.16b1149; CHECK-SVE-NEXT: ret1150;1151; CHECK-GI-LABEL: amull_extvec_v2i32_v2i64:1152; CHECK-GI: // %bb.0:1153; CHECK-GI-NEXT: adrp x8, .LCPI43_01154; CHECK-GI-NEXT: movi v2.2d, #0x000000ffffffff1155; CHECK-GI-NEXT: ldr d1, [x8, :lo12:.LCPI43_0]1156; CHECK-GI-NEXT: umull v0.2d, v0.2s, v1.2s1157; CHECK-GI-NEXT: and v0.16b, v0.16b, v2.16b1158; CHECK-GI-NEXT: ret1159 %tmp3 = zext <2 x i32> %arg to <2 x i64>1160 %tmp4 = mul <2 x i64> %tmp3, <i64 1234, i64 1234>1161 %and = and <2 x i64> %tmp4, <i64 4294967295, i64 4294967295>1162 ret <2 x i64> %and1163}1164 1165define i16 @smullWithInconsistentExtensions(<8 x i8> %x, <8 x i8> %y) {1166; If one operand has a zero-extend and the other a sign-extend, smull1167; cannot be used.1168; CHECK-LABEL: smullWithInconsistentExtensions:1169; CHECK: // %bb.0:1170; CHECK-NEXT: sshll v0.8h, v0.8b, #01171; CHECK-NEXT: ushll v1.8h, v1.8b, #01172; CHECK-NEXT: mul v0.8h, v0.8h, v1.8h1173; CHECK-NEXT: umov w0, v0.h[0]1174; CHECK-NEXT: ret1175 %s = sext <8 x i8> %x to <8 x i16>1176 %z = zext <8 x i8> %y to <8 x i16>1177 %m = mul <8 x i16> %s, %z1178 %r = extractelement <8 x i16> %m, i32 01179 ret i16 %r1180}1181 1182define <8 x i16> @smull_extended_vector_operand(<8 x i16> %v) {1183; CHECK-LABEL: smull_extended_vector_operand:1184; CHECK: // %bb.0: // %entry1185; CHECK-NEXT: movi v1.4s, #139, lsl #81186; CHECK-NEXT: sshll v2.4s, v0.4h, #01187; CHECK-NEXT: sshll2 v0.4s, v0.8h, #01188; CHECK-NEXT: mul v2.4s, v2.4s, v1.4s1189; CHECK-NEXT: mul v1.4s, v0.4s, v1.4s1190; CHECK-NEXT: shrn v0.4h, v2.4s, #11191; CHECK-NEXT: shrn2 v0.8h, v1.4s, #11192; CHECK-NEXT: ret1193entry:1194%0 = sext <8 x i16> %v to <8 x i32>1195%1 = mul <8 x i32> %0, <i32 35584, i32 35584, i32 35584, i32 35584, i32 35584, i32 35584, i32 35584, i32 35584>1196%2 = lshr <8 x i32> %1, <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>1197%3 = trunc <8 x i32> %2 to <8 x i16>1198ret <8 x i16> %31199 1200}1201 1202define void @distribute(ptr %dst, ptr %src, i32 %mul) nounwind {1203; CHECK-NEON-LABEL: distribute:1204; CHECK-NEON: // %bb.0: // %entry1205; CHECK-NEON-NEXT: ldr q0, [x1]1206; CHECK-NEON-NEXT: dup v1.8b, w21207; CHECK-NEON-NEXT: mov d2, v0.d[1]1208; CHECK-NEON-NEXT: umull v2.8h, v2.8b, v1.8b1209; CHECK-NEON-NEXT: umlal v2.8h, v0.8b, v1.8b1210; CHECK-NEON-NEXT: str q2, [x0]1211; CHECK-NEON-NEXT: ret1212;1213; CHECK-SVE-LABEL: distribute:1214; CHECK-SVE: // %bb.0: // %entry1215; CHECK-SVE-NEXT: ldr q0, [x1]1216; CHECK-SVE-NEXT: dup v1.8b, w21217; CHECK-SVE-NEXT: mov d2, v0.d[1]1218; CHECK-SVE-NEXT: umull v2.8h, v2.8b, v1.8b1219; CHECK-SVE-NEXT: umlal v2.8h, v0.8b, v1.8b1220; CHECK-SVE-NEXT: str q2, [x0]1221; CHECK-SVE-NEXT: ret1222;1223; CHECK-GI-LABEL: distribute:1224; CHECK-GI: // %bb.0: // %entry1225; CHECK-GI-NEXT: ldr q0, [x1]1226; CHECK-GI-NEXT: dup v1.8b, w21227; CHECK-GI-NEXT: ushll v2.8h, v0.8b, #01228; CHECK-GI-NEXT: ushll v1.8h, v1.8b, #01229; CHECK-GI-NEXT: uaddw2 v0.8h, v2.8h, v0.16b1230; CHECK-GI-NEXT: mul v0.8h, v0.8h, v1.8h1231; CHECK-GI-NEXT: str q0, [x0]1232; CHECK-GI-NEXT: ret1233entry:1234 %0 = trunc i32 %mul to i81235 %1 = insertelement <8 x i8> undef, i8 %0, i32 01236 %2 = shufflevector <8 x i8> %1, <8 x i8> undef, <8 x i32> zeroinitializer1237 %3 = load <16 x i8>, ptr %src, align 11238 %4 = bitcast <16 x i8> %3 to <2 x double>1239 %5 = extractelement <2 x double> %4, i32 11240 %6 = bitcast double %5 to <8 x i8>1241 %7 = zext <8 x i8> %6 to <8 x i16>1242 %8 = zext <8 x i8> %2 to <8 x i16>1243 %9 = extractelement <2 x double> %4, i32 01244 %10 = bitcast double %9 to <8 x i8>1245 %11 = zext <8 x i8> %10 to <8 x i16>1246 %12 = add <8 x i16> %7, %111247 %13 = mul <8 x i16> %12, %81248 store <8 x i16> %13, ptr %dst, align 21249 ret void1250}1251 1252define <16 x i16> @umull2_i8(<16 x i8> %arg1, <16 x i8> %arg2) {1253; CHECK-NEON-LABEL: umull2_i8:1254; CHECK-NEON: // %bb.0:1255; CHECK-NEON-NEXT: umull2 v2.8h, v0.16b, v1.16b1256; CHECK-NEON-NEXT: umull v0.8h, v0.8b, v1.8b1257; CHECK-NEON-NEXT: mov v1.16b, v2.16b1258; CHECK-NEON-NEXT: ret1259;1260; CHECK-SVE-LABEL: umull2_i8:1261; CHECK-SVE: // %bb.0:1262; CHECK-SVE-NEXT: umull2 v2.8h, v0.16b, v1.16b1263; CHECK-SVE-NEXT: umull v0.8h, v0.8b, v1.8b1264; CHECK-SVE-NEXT: mov v1.16b, v2.16b1265; CHECK-SVE-NEXT: ret1266;1267; CHECK-GI-LABEL: umull2_i8:1268; CHECK-GI: // %bb.0:1269; CHECK-GI-NEXT: umull v2.8h, v0.8b, v1.8b1270; CHECK-GI-NEXT: umull2 v1.8h, v0.16b, v1.16b1271; CHECK-GI-NEXT: mov v0.16b, v2.16b1272; CHECK-GI-NEXT: ret1273 %arg1_ext = zext <16 x i8> %arg1 to <16 x i16>1274 %arg2_ext = zext <16 x i8> %arg2 to <16 x i16>1275 %mul = mul <16 x i16> %arg1_ext, %arg2_ext1276 ret <16 x i16> %mul1277}1278 1279define <16 x i16> @smull2_i8(<16 x i8> %arg1, <16 x i8> %arg2) {1280; CHECK-NEON-LABEL: smull2_i8:1281; CHECK-NEON: // %bb.0:1282; CHECK-NEON-NEXT: smull2 v2.8h, v0.16b, v1.16b1283; CHECK-NEON-NEXT: smull v0.8h, v0.8b, v1.8b1284; CHECK-NEON-NEXT: mov v1.16b, v2.16b1285; CHECK-NEON-NEXT: ret1286;1287; CHECK-SVE-LABEL: smull2_i8:1288; CHECK-SVE: // %bb.0:1289; CHECK-SVE-NEXT: smull2 v2.8h, v0.16b, v1.16b1290; CHECK-SVE-NEXT: smull v0.8h, v0.8b, v1.8b1291; CHECK-SVE-NEXT: mov v1.16b, v2.16b1292; CHECK-SVE-NEXT: ret1293;1294; CHECK-GI-LABEL: smull2_i8:1295; CHECK-GI: // %bb.0:1296; CHECK-GI-NEXT: smull v2.8h, v0.8b, v1.8b1297; CHECK-GI-NEXT: smull2 v1.8h, v0.16b, v1.16b1298; CHECK-GI-NEXT: mov v0.16b, v2.16b1299; CHECK-GI-NEXT: ret1300 %arg1_ext = sext <16 x i8> %arg1 to <16 x i16>1301 %arg2_ext = sext <16 x i8> %arg2 to <16 x i16>1302 %mul = mul <16 x i16> %arg1_ext, %arg2_ext1303 ret <16 x i16> %mul1304}1305 1306define <8 x i32> @umull2_i16(<8 x i16> %arg1, <8 x i16> %arg2) {1307; CHECK-NEON-LABEL: umull2_i16:1308; CHECK-NEON: // %bb.0:1309; CHECK-NEON-NEXT: umull2 v2.4s, v0.8h, v1.8h1310; CHECK-NEON-NEXT: umull v0.4s, v0.4h, v1.4h1311; CHECK-NEON-NEXT: mov v1.16b, v2.16b1312; CHECK-NEON-NEXT: ret1313;1314; CHECK-SVE-LABEL: umull2_i16:1315; CHECK-SVE: // %bb.0:1316; CHECK-SVE-NEXT: umull2 v2.4s, v0.8h, v1.8h1317; CHECK-SVE-NEXT: umull v0.4s, v0.4h, v1.4h1318; CHECK-SVE-NEXT: mov v1.16b, v2.16b1319; CHECK-SVE-NEXT: ret1320;1321; CHECK-GI-LABEL: umull2_i16:1322; CHECK-GI: // %bb.0:1323; CHECK-GI-NEXT: umull v2.4s, v0.4h, v1.4h1324; CHECK-GI-NEXT: umull2 v1.4s, v0.8h, v1.8h1325; CHECK-GI-NEXT: mov v0.16b, v2.16b1326; CHECK-GI-NEXT: ret1327 %arg1_ext = zext <8 x i16> %arg1 to <8 x i32>1328 %arg2_ext = zext <8 x i16> %arg2 to <8 x i32>1329 %mul = mul <8 x i32> %arg1_ext, %arg2_ext1330 ret <8 x i32> %mul1331}1332 1333define <8 x i32> @smull2_i16(<8 x i16> %arg1, <8 x i16> %arg2) {1334; CHECK-NEON-LABEL: smull2_i16:1335; CHECK-NEON: // %bb.0:1336; CHECK-NEON-NEXT: smull2 v2.4s, v0.8h, v1.8h1337; CHECK-NEON-NEXT: smull v0.4s, v0.4h, v1.4h1338; CHECK-NEON-NEXT: mov v1.16b, v2.16b1339; CHECK-NEON-NEXT: ret1340;1341; CHECK-SVE-LABEL: smull2_i16:1342; CHECK-SVE: // %bb.0:1343; CHECK-SVE-NEXT: smull2 v2.4s, v0.8h, v1.8h1344; CHECK-SVE-NEXT: smull v0.4s, v0.4h, v1.4h1345; CHECK-SVE-NEXT: mov v1.16b, v2.16b1346; CHECK-SVE-NEXT: ret1347;1348; CHECK-GI-LABEL: smull2_i16:1349; CHECK-GI: // %bb.0:1350; CHECK-GI-NEXT: smull v2.4s, v0.4h, v1.4h1351; CHECK-GI-NEXT: smull2 v1.4s, v0.8h, v1.8h1352; CHECK-GI-NEXT: mov v0.16b, v2.16b1353; CHECK-GI-NEXT: ret1354 %arg1_ext = sext <8 x i16> %arg1 to <8 x i32>1355 %arg2_ext = sext <8 x i16> %arg2 to <8 x i32>1356 %mul = mul <8 x i32> %arg1_ext, %arg2_ext1357 ret <8 x i32> %mul1358}1359 1360define <4 x i64> @umull2_i32(<4 x i32> %arg1, <4 x i32> %arg2) {1361; CHECK-NEON-LABEL: umull2_i32:1362; CHECK-NEON: // %bb.0:1363; CHECK-NEON-NEXT: umull2 v2.2d, v0.4s, v1.4s1364; CHECK-NEON-NEXT: umull v0.2d, v0.2s, v1.2s1365; CHECK-NEON-NEXT: mov v1.16b, v2.16b1366; CHECK-NEON-NEXT: ret1367;1368; CHECK-SVE-LABEL: umull2_i32:1369; CHECK-SVE: // %bb.0:1370; CHECK-SVE-NEXT: umull2 v2.2d, v0.4s, v1.4s1371; CHECK-SVE-NEXT: umull v0.2d, v0.2s, v1.2s1372; CHECK-SVE-NEXT: mov v1.16b, v2.16b1373; CHECK-SVE-NEXT: ret1374;1375; CHECK-GI-LABEL: umull2_i32:1376; CHECK-GI: // %bb.0:1377; CHECK-GI-NEXT: umull v2.2d, v0.2s, v1.2s1378; CHECK-GI-NEXT: umull2 v1.2d, v0.4s, v1.4s1379; CHECK-GI-NEXT: mov v0.16b, v2.16b1380; CHECK-GI-NEXT: ret1381 %arg1_ext = zext <4 x i32> %arg1 to <4 x i64>1382 %arg2_ext = zext <4 x i32> %arg2 to <4 x i64>1383 %mul = mul <4 x i64> %arg1_ext, %arg2_ext1384 ret <4 x i64> %mul1385}1386 1387define <4 x i64> @smull2_i32(<4 x i32> %arg1, <4 x i32> %arg2) {1388; CHECK-NEON-LABEL: smull2_i32:1389; CHECK-NEON: // %bb.0:1390; CHECK-NEON-NEXT: smull2 v2.2d, v0.4s, v1.4s1391; CHECK-NEON-NEXT: smull v0.2d, v0.2s, v1.2s1392; CHECK-NEON-NEXT: mov v1.16b, v2.16b1393; CHECK-NEON-NEXT: ret1394;1395; CHECK-SVE-LABEL: smull2_i32:1396; CHECK-SVE: // %bb.0:1397; CHECK-SVE-NEXT: smull2 v2.2d, v0.4s, v1.4s1398; CHECK-SVE-NEXT: smull v0.2d, v0.2s, v1.2s1399; CHECK-SVE-NEXT: mov v1.16b, v2.16b1400; CHECK-SVE-NEXT: ret1401;1402; CHECK-GI-LABEL: smull2_i32:1403; CHECK-GI: // %bb.0:1404; CHECK-GI-NEXT: smull v2.2d, v0.2s, v1.2s1405; CHECK-GI-NEXT: smull2 v1.2d, v0.4s, v1.4s1406; CHECK-GI-NEXT: mov v0.16b, v2.16b1407; CHECK-GI-NEXT: ret1408 %arg1_ext = sext <4 x i32> %arg1 to <4 x i64>1409 %arg2_ext = sext <4 x i32> %arg2 to <4 x i64>1410 %mul = mul <4 x i64> %arg1_ext, %arg2_ext1411 ret <4 x i64> %mul1412}1413 1414define <16 x i16> @amull2_i8(<16 x i8> %arg1, <16 x i8> %arg2) {1415; CHECK-NEON-LABEL: amull2_i8:1416; CHECK-NEON: // %bb.0:1417; CHECK-NEON-NEXT: smull v2.8h, v0.8b, v1.8b1418; CHECK-NEON-NEXT: smull2 v1.8h, v0.16b, v1.16b1419; CHECK-NEON-NEXT: bic v2.8h, #255, lsl #81420; CHECK-NEON-NEXT: bic v1.8h, #255, lsl #81421; CHECK-NEON-NEXT: mov v0.16b, v2.16b1422; CHECK-NEON-NEXT: ret1423;1424; CHECK-SVE-LABEL: amull2_i8:1425; CHECK-SVE: // %bb.0:1426; CHECK-SVE-NEXT: smull v2.8h, v0.8b, v1.8b1427; CHECK-SVE-NEXT: smull2 v1.8h, v0.16b, v1.16b1428; CHECK-SVE-NEXT: bic v2.8h, #255, lsl #81429; CHECK-SVE-NEXT: bic v1.8h, #255, lsl #81430; CHECK-SVE-NEXT: mov v0.16b, v2.16b1431; CHECK-SVE-NEXT: ret1432;1433; CHECK-GI-LABEL: amull2_i8:1434; CHECK-GI: // %bb.0:1435; CHECK-GI-NEXT: movi v2.2d, #0xff00ff00ff00ff1436; CHECK-GI-NEXT: umull v3.8h, v0.8b, v1.8b1437; CHECK-GI-NEXT: umull2 v1.8h, v0.16b, v1.16b1438; CHECK-GI-NEXT: and v0.16b, v3.16b, v2.16b1439; CHECK-GI-NEXT: and v1.16b, v1.16b, v2.16b1440; CHECK-GI-NEXT: ret1441 %arg1_ext = zext <16 x i8> %arg1 to <16 x i16>1442 %arg2_ext = zext <16 x i8> %arg2 to <16 x i16>1443 %mul = mul <16 x i16> %arg1_ext, %arg2_ext1444 %and = and <16 x i16> %mul, <i16 255, i16 255, i16 255, i16 255, i16 255, i16 255, i16 255, i16 255, i16 255, i16 255, i16 255, i16 255, i16 255, i16 255, i16 255, i16 255>1445 ret <16 x i16> %and1446}1447 1448define <8 x i32> @amull2_i16(<8 x i16> %arg1, <8 x i16> %arg2) {1449; CHECK-NEON-LABEL: amull2_i16:1450; CHECK-NEON: // %bb.0:1451; CHECK-NEON-NEXT: movi v2.2d, #0x00ffff0000ffff1452; CHECK-NEON-NEXT: smull v3.4s, v0.4h, v1.4h1453; CHECK-NEON-NEXT: smull2 v0.4s, v0.8h, v1.8h1454; CHECK-NEON-NEXT: and v1.16b, v0.16b, v2.16b1455; CHECK-NEON-NEXT: and v0.16b, v3.16b, v2.16b1456; CHECK-NEON-NEXT: ret1457;1458; CHECK-SVE-LABEL: amull2_i16:1459; CHECK-SVE: // %bb.0:1460; CHECK-SVE-NEXT: movi v2.2d, #0x00ffff0000ffff1461; CHECK-SVE-NEXT: smull v3.4s, v0.4h, v1.4h1462; CHECK-SVE-NEXT: smull2 v0.4s, v0.8h, v1.8h1463; CHECK-SVE-NEXT: and v1.16b, v0.16b, v2.16b1464; CHECK-SVE-NEXT: and v0.16b, v3.16b, v2.16b1465; CHECK-SVE-NEXT: ret1466;1467; CHECK-GI-LABEL: amull2_i16:1468; CHECK-GI: // %bb.0:1469; CHECK-GI-NEXT: movi v2.2d, #0x00ffff0000ffff1470; CHECK-GI-NEXT: umull v3.4s, v0.4h, v1.4h1471; CHECK-GI-NEXT: umull2 v1.4s, v0.8h, v1.8h1472; CHECK-GI-NEXT: and v0.16b, v3.16b, v2.16b1473; CHECK-GI-NEXT: and v1.16b, v1.16b, v2.16b1474; CHECK-GI-NEXT: ret1475 %arg1_ext = zext <8 x i16> %arg1 to <8 x i32>1476 %arg2_ext = zext <8 x i16> %arg2 to <8 x i32>1477 %mul = mul <8 x i32> %arg1_ext, %arg2_ext1478 %and = and <8 x i32> %mul, <i32 65535, i32 65535, i32 65535, i32 65535, i32 65535, i32 65535, i32 65535, i32 65535>1479 ret <8 x i32> %and1480}1481 1482define <4 x i64> @amull2_i32(<4 x i32> %arg1, <4 x i32> %arg2) {1483; CHECK-NEON-LABEL: amull2_i32:1484; CHECK-NEON: // %bb.0:1485; CHECK-NEON-NEXT: movi v2.2d, #0x000000ffffffff1486; CHECK-NEON-NEXT: smull v3.2d, v0.2s, v1.2s1487; CHECK-NEON-NEXT: smull2 v0.2d, v0.4s, v1.4s1488; CHECK-NEON-NEXT: and v1.16b, v0.16b, v2.16b1489; CHECK-NEON-NEXT: and v0.16b, v3.16b, v2.16b1490; CHECK-NEON-NEXT: ret1491;1492; CHECK-SVE-LABEL: amull2_i32:1493; CHECK-SVE: // %bb.0:1494; CHECK-SVE-NEXT: movi v2.2d, #0x000000ffffffff1495; CHECK-SVE-NEXT: smull v3.2d, v0.2s, v1.2s1496; CHECK-SVE-NEXT: smull2 v0.2d, v0.4s, v1.4s1497; CHECK-SVE-NEXT: and v1.16b, v0.16b, v2.16b1498; CHECK-SVE-NEXT: and v0.16b, v3.16b, v2.16b1499; CHECK-SVE-NEXT: ret1500;1501; CHECK-GI-LABEL: amull2_i32:1502; CHECK-GI: // %bb.0:1503; CHECK-GI-NEXT: movi v2.2d, #0x000000ffffffff1504; CHECK-GI-NEXT: umull v3.2d, v0.2s, v1.2s1505; CHECK-GI-NEXT: umull2 v1.2d, v0.4s, v1.4s1506; CHECK-GI-NEXT: and v0.16b, v3.16b, v2.16b1507; CHECK-GI-NEXT: and v1.16b, v1.16b, v2.16b1508; CHECK-GI-NEXT: ret1509 %arg1_ext = zext <4 x i32> %arg1 to <4 x i64>1510 %arg2_ext = zext <4 x i32> %arg2 to <4 x i64>1511 %mul = mul <4 x i64> %arg1_ext, %arg2_ext1512 %and = and <4 x i64> %mul, <i64 4294967295, i64 4294967295, i64 4294967295, i64 4294967295>1513 ret <4 x i64> %and1514}1515 1516 1517define <8 x i16> @umull_and_v8i16(<8 x i8> %src1, <8 x i16> %src2) {1518; CHECK-NEON-LABEL: umull_and_v8i16:1519; CHECK-NEON: // %bb.0: // %entry1520; CHECK-NEON-NEXT: bic v1.8h, #255, lsl #81521; CHECK-NEON-NEXT: xtn v1.8b, v1.8h1522; CHECK-NEON-NEXT: umull v0.8h, v0.8b, v1.8b1523; CHECK-NEON-NEXT: ret1524;1525; CHECK-SVE-LABEL: umull_and_v8i16:1526; CHECK-SVE: // %bb.0: // %entry1527; CHECK-SVE-NEXT: bic v1.8h, #255, lsl #81528; CHECK-SVE-NEXT: xtn v1.8b, v1.8h1529; CHECK-SVE-NEXT: umull v0.8h, v0.8b, v1.8b1530; CHECK-SVE-NEXT: ret1531;1532; CHECK-GI-LABEL: umull_and_v8i16:1533; CHECK-GI: // %bb.0: // %entry1534; CHECK-GI-NEXT: movi v2.2d, #0xff00ff00ff00ff1535; CHECK-GI-NEXT: and v1.16b, v1.16b, v2.16b1536; CHECK-GI-NEXT: xtn v1.8b, v1.8h1537; CHECK-GI-NEXT: umull v0.8h, v0.8b, v1.8b1538; CHECK-GI-NEXT: ret1539entry:1540 %in1 = zext <8 x i8> %src1 to <8 x i16>1541 %in2 = and <8 x i16> %src2, <i16 255, i16 255, i16 255, i16 255, i16 255, i16 255, i16 255, i16 255>1542 %out = mul nsw <8 x i16> %in1, %in21543 ret <8 x i16> %out1544}1545 1546define <8 x i16> @umull_and_v8i16_c(<8 x i8> %src1, <8 x i16> %src2) {1547; CHECK-NEON-LABEL: umull_and_v8i16_c:1548; CHECK-NEON: // %bb.0: // %entry1549; CHECK-NEON-NEXT: bic v1.8h, #255, lsl #81550; CHECK-NEON-NEXT: xtn v1.8b, v1.8h1551; CHECK-NEON-NEXT: umull v0.8h, v1.8b, v0.8b1552; CHECK-NEON-NEXT: ret1553;1554; CHECK-SVE-LABEL: umull_and_v8i16_c:1555; CHECK-SVE: // %bb.0: // %entry1556; CHECK-SVE-NEXT: bic v1.8h, #255, lsl #81557; CHECK-SVE-NEXT: xtn v1.8b, v1.8h1558; CHECK-SVE-NEXT: umull v0.8h, v1.8b, v0.8b1559; CHECK-SVE-NEXT: ret1560;1561; CHECK-GI-LABEL: umull_and_v8i16_c:1562; CHECK-GI: // %bb.0: // %entry1563; CHECK-GI-NEXT: movi v2.2d, #0xff00ff00ff00ff1564; CHECK-GI-NEXT: and v1.16b, v1.16b, v2.16b1565; CHECK-GI-NEXT: xtn v1.8b, v1.8h1566; CHECK-GI-NEXT: umull v0.8h, v1.8b, v0.8b1567; CHECK-GI-NEXT: ret1568entry:1569 %in1 = zext <8 x i8> %src1 to <8 x i16>1570 %in2 = and <8 x i16> %src2, <i16 255, i16 255, i16 255, i16 255, i16 255, i16 255, i16 255, i16 255>1571 %out = mul nsw <8 x i16> %in2, %in11572 ret <8 x i16> %out1573}1574 1575define <8 x i16> @umull_and256_v8i16(<8 x i8> %src1, <8 x i16> %src2) {1576; CHECK-LABEL: umull_and256_v8i16:1577; CHECK: // %bb.0: // %entry1578; CHECK-NEXT: movi v2.8h, #1, lsl #81579; CHECK-NEXT: ushll v0.8h, v0.8b, #01580; CHECK-NEXT: and v1.16b, v1.16b, v2.16b1581; CHECK-NEXT: mul v0.8h, v0.8h, v1.8h1582; CHECK-NEXT: ret1583entry:1584 %in1 = zext <8 x i8> %src1 to <8 x i16>1585 %in2 = and <8 x i16> %src2, <i16 256, i16 256, i16 256, i16 256, i16 256, i16 256, i16 256, i16 256>1586 %out = mul nsw <8 x i16> %in1, %in21587 ret <8 x i16> %out1588}1589 1590define <8 x i16> @umull_andconst_v8i16(<8 x i8> %src1, <8 x i16> %src2) {1591; CHECK-NEON-LABEL: umull_andconst_v8i16:1592; CHECK-NEON: // %bb.0: // %entry1593; CHECK-NEON-NEXT: movi v1.2d, #0xffffffffffffffff1594; CHECK-NEON-NEXT: umull v0.8h, v0.8b, v1.8b1595; CHECK-NEON-NEXT: ret1596;1597; CHECK-SVE-LABEL: umull_andconst_v8i16:1598; CHECK-SVE: // %bb.0: // %entry1599; CHECK-SVE-NEXT: movi v1.2d, #0xffffffffffffffff1600; CHECK-SVE-NEXT: umull v0.8h, v0.8b, v1.8b1601; CHECK-SVE-NEXT: ret1602;1603; CHECK-GI-LABEL: umull_andconst_v8i16:1604; CHECK-GI: // %bb.0: // %entry1605; CHECK-GI-NEXT: movi d1, #0xffffffffffffffff1606; CHECK-GI-NEXT: umull v0.8h, v0.8b, v1.8b1607; CHECK-GI-NEXT: ret1608entry:1609 %in1 = zext <8 x i8> %src1 to <8 x i16>1610 %out = mul nsw <8 x i16> %in1, <i16 255, i16 255, i16 255, i16 255, i16 255, i16 255, i16 255, i16 255>1611 ret <8 x i16> %out1612}1613 1614define <8 x i16> @umull_smaller_v8i16(<8 x i4> %src1, <8 x i16> %src2) {1615; CHECK-NEON-LABEL: umull_smaller_v8i16:1616; CHECK-NEON: // %bb.0: // %entry1617; CHECK-NEON-NEXT: movi v2.8b, #151618; CHECK-NEON-NEXT: bic v1.8h, #255, lsl #81619; CHECK-NEON-NEXT: xtn v1.8b, v1.8h1620; CHECK-NEON-NEXT: and v0.8b, v0.8b, v2.8b1621; CHECK-NEON-NEXT: umull v0.8h, v0.8b, v1.8b1622; CHECK-NEON-NEXT: ret1623;1624; CHECK-SVE-LABEL: umull_smaller_v8i16:1625; CHECK-SVE: // %bb.0: // %entry1626; CHECK-SVE-NEXT: movi v2.8b, #151627; CHECK-SVE-NEXT: bic v1.8h, #255, lsl #81628; CHECK-SVE-NEXT: xtn v1.8b, v1.8h1629; CHECK-SVE-NEXT: and v0.8b, v0.8b, v2.8b1630; CHECK-SVE-NEXT: umull v0.8h, v0.8b, v1.8b1631; CHECK-SVE-NEXT: ret1632;1633; CHECK-GI-LABEL: umull_smaller_v8i16:1634; CHECK-GI: // %bb.0: // %entry1635; CHECK-GI-NEXT: movi v2.2d, #0xff00ff00ff00ff1636; CHECK-GI-NEXT: movi v3.8h, #151637; CHECK-GI-NEXT: ushll v0.8h, v0.8b, #01638; CHECK-GI-NEXT: and v0.16b, v0.16b, v3.16b1639; CHECK-GI-NEXT: and v1.16b, v1.16b, v2.16b1640; CHECK-GI-NEXT: mul v0.8h, v0.8h, v1.8h1641; CHECK-GI-NEXT: ret1642entry:1643 %in1 = zext <8 x i4> %src1 to <8 x i16>1644 %in2 = and <8 x i16> %src2, <i16 255, i16 255, i16 255, i16 255, i16 255, i16 255, i16 255, i16 255>1645 %out = mul nsw <8 x i16> %in1, %in21646 ret <8 x i16> %out1647}1648 1649define <4 x i32> @umull_and_v4i32(<4 x i16> %src1, <4 x i32> %src2) {1650; CHECK-LABEL: umull_and_v4i32:1651; CHECK: // %bb.0: // %entry1652; CHECK-NEXT: movi v2.2d, #0x0000ff000000ff1653; CHECK-NEXT: and v1.16b, v1.16b, v2.16b1654; CHECK-NEXT: xtn v1.4h, v1.4s1655; CHECK-NEXT: umull v0.4s, v0.4h, v1.4h1656; CHECK-NEXT: ret1657entry:1658 %in1 = zext <4 x i16> %src1 to <4 x i32>1659 %in2 = and <4 x i32> %src2, <i32 255, i32 255, i32 255, i32 255>1660 %out = mul nsw <4 x i32> %in1, %in21661 ret <4 x i32> %out1662}1663 1664define <8 x i32> @umull_and_v8i32(<8 x i16> %src1, <8 x i32> %src2) {1665; CHECK-NEON-LABEL: umull_and_v8i32:1666; CHECK-NEON: // %bb.0: // %entry1667; CHECK-NEON-NEXT: movi v3.2d, #0x0000ff000000ff1668; CHECK-NEON-NEXT: and v2.16b, v2.16b, v3.16b1669; CHECK-NEON-NEXT: and v1.16b, v1.16b, v3.16b1670; CHECK-NEON-NEXT: uzp1 v2.8h, v1.8h, v2.8h1671; CHECK-NEON-NEXT: umull2 v1.4s, v0.8h, v2.8h1672; CHECK-NEON-NEXT: umull v0.4s, v0.4h, v2.4h1673; CHECK-NEON-NEXT: ret1674;1675; CHECK-SVE-LABEL: umull_and_v8i32:1676; CHECK-SVE: // %bb.0: // %entry1677; CHECK-SVE-NEXT: movi v3.2d, #0x0000ff000000ff1678; CHECK-SVE-NEXT: and v2.16b, v2.16b, v3.16b1679; CHECK-SVE-NEXT: and v1.16b, v1.16b, v3.16b1680; CHECK-SVE-NEXT: uzp1 v2.8h, v1.8h, v2.8h1681; CHECK-SVE-NEXT: umull2 v1.4s, v0.8h, v2.8h1682; CHECK-SVE-NEXT: umull v0.4s, v0.4h, v2.4h1683; CHECK-SVE-NEXT: ret1684;1685; CHECK-GI-LABEL: umull_and_v8i32:1686; CHECK-GI: // %bb.0: // %entry1687; CHECK-GI-NEXT: movi v3.2d, #0x0000ff000000ff1688; CHECK-GI-NEXT: and v1.16b, v1.16b, v3.16b1689; CHECK-GI-NEXT: and v2.16b, v2.16b, v3.16b1690; CHECK-GI-NEXT: mov d3, v0.d[1]1691; CHECK-GI-NEXT: xtn v1.4h, v1.4s1692; CHECK-GI-NEXT: xtn v2.4h, v2.4s1693; CHECK-GI-NEXT: umull v0.4s, v0.4h, v1.4h1694; CHECK-GI-NEXT: umull v1.4s, v3.4h, v2.4h1695; CHECK-GI-NEXT: ret1696entry:1697 %in1 = zext <8 x i16> %src1 to <8 x i32>1698 %in2 = and <8 x i32> %src2, <i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255>1699 %out = mul nsw <8 x i32> %in1, %in21700 ret <8 x i32> %out1701}1702 1703define <8 x i32> @umull_and_v8i32_dup(<8 x i16> %src1, i32 %src2) {1704; CHECK-NEON-LABEL: umull_and_v8i32_dup:1705; CHECK-NEON: // %bb.0: // %entry1706; CHECK-NEON-NEXT: and w8, w0, #0xff1707; CHECK-NEON-NEXT: dup v2.8h, w81708; CHECK-NEON-NEXT: umull2 v1.4s, v0.8h, v2.8h1709; CHECK-NEON-NEXT: umull v0.4s, v0.4h, v2.4h1710; CHECK-NEON-NEXT: ret1711;1712; CHECK-SVE-LABEL: umull_and_v8i32_dup:1713; CHECK-SVE: // %bb.0: // %entry1714; CHECK-SVE-NEXT: and w8, w0, #0xff1715; CHECK-SVE-NEXT: dup v2.8h, w81716; CHECK-SVE-NEXT: umull2 v1.4s, v0.8h, v2.8h1717; CHECK-SVE-NEXT: umull v0.4s, v0.4h, v2.4h1718; CHECK-SVE-NEXT: ret1719;1720; CHECK-GI-LABEL: umull_and_v8i32_dup:1721; CHECK-GI: // %bb.0: // %entry1722; CHECK-GI-NEXT: and w8, w0, #0xff1723; CHECK-GI-NEXT: mov d2, v0.d[1]1724; CHECK-GI-NEXT: dup v1.4s, w81725; CHECK-GI-NEXT: xtn v1.4h, v1.4s1726; CHECK-GI-NEXT: umull v0.4s, v0.4h, v1.4h1727; CHECK-GI-NEXT: umull v1.4s, v2.4h, v1.4h1728; CHECK-GI-NEXT: ret1729entry:1730 %in1 = zext <8 x i16> %src1 to <8 x i32>1731 %in2 = and i32 %src2, 2551732 %broadcast.splatinsert = insertelement <8 x i32> undef, i32 %in2, i64 01733 %broadcast.splat = shufflevector <8 x i32> %broadcast.splatinsert, <8 x i32> undef, <8 x i32> zeroinitializer1734 %out = mul nsw <8 x i32> %in1, %broadcast.splat1735 ret <8 x i32> %out1736}1737 1738define <2 x i64> @umull_and_v2i64(<2 x i32> %src1, <2 x i64> %src2) {1739; CHECK-LABEL: umull_and_v2i64:1740; CHECK: // %bb.0: // %entry1741; CHECK-NEXT: movi v2.2d, #0x000000000000ff1742; CHECK-NEXT: and v1.16b, v1.16b, v2.16b1743; CHECK-NEXT: xtn v1.2s, v1.2d1744; CHECK-NEXT: umull v0.2d, v0.2s, v1.2s1745; CHECK-NEXT: ret1746entry:1747 %in1 = zext <2 x i32> %src1 to <2 x i64>1748 %in2 = and <2 x i64> %src2, <i64 255, i64 255>1749 %out = mul nsw <2 x i64> %in1, %in21750 ret <2 x i64> %out1751}1752 1753define <4 x i64> @umull_and_v4i64(<4 x i32> %src1, <4 x i64> %src2) {1754; CHECK-NEON-LABEL: umull_and_v4i64:1755; CHECK-NEON: // %bb.0: // %entry1756; CHECK-NEON-NEXT: movi v3.2d, #0x000000000000ff1757; CHECK-NEON-NEXT: and v2.16b, v2.16b, v3.16b1758; CHECK-NEON-NEXT: and v1.16b, v1.16b, v3.16b1759; CHECK-NEON-NEXT: uzp1 v2.4s, v1.4s, v2.4s1760; CHECK-NEON-NEXT: umull2 v1.2d, v0.4s, v2.4s1761; CHECK-NEON-NEXT: umull v0.2d, v0.2s, v2.2s1762; CHECK-NEON-NEXT: ret1763;1764; CHECK-SVE-LABEL: umull_and_v4i64:1765; CHECK-SVE: // %bb.0: // %entry1766; CHECK-SVE-NEXT: movi v3.2d, #0x000000000000ff1767; CHECK-SVE-NEXT: and v2.16b, v2.16b, v3.16b1768; CHECK-SVE-NEXT: and v1.16b, v1.16b, v3.16b1769; CHECK-SVE-NEXT: uzp1 v2.4s, v1.4s, v2.4s1770; CHECK-SVE-NEXT: umull2 v1.2d, v0.4s, v2.4s1771; CHECK-SVE-NEXT: umull v0.2d, v0.2s, v2.2s1772; CHECK-SVE-NEXT: ret1773;1774; CHECK-GI-LABEL: umull_and_v4i64:1775; CHECK-GI: // %bb.0: // %entry1776; CHECK-GI-NEXT: movi v3.2d, #0x000000000000ff1777; CHECK-GI-NEXT: and v1.16b, v1.16b, v3.16b1778; CHECK-GI-NEXT: and v2.16b, v2.16b, v3.16b1779; CHECK-GI-NEXT: mov d3, v0.d[1]1780; CHECK-GI-NEXT: xtn v1.2s, v1.2d1781; CHECK-GI-NEXT: xtn v2.2s, v2.2d1782; CHECK-GI-NEXT: umull v0.2d, v0.2s, v1.2s1783; CHECK-GI-NEXT: umull v1.2d, v3.2s, v2.2s1784; CHECK-GI-NEXT: ret1785entry:1786 %in1 = zext <4 x i32> %src1 to <4 x i64>1787 %in2 = and <4 x i64> %src2, <i64 255, i64 255, i64 255, i64 255>1788 %out = mul nsw <4 x i64> %in1, %in21789 ret <4 x i64> %out1790}1791 1792define <4 x i64> @umull_and_v4i64_dup(<4 x i32> %src1, i64 %src2) {1793; CHECK-NEON-LABEL: umull_and_v4i64_dup:1794; CHECK-NEON: // %bb.0: // %entry1795; CHECK-NEON-NEXT: and w8, w0, #0xff1796; CHECK-NEON-NEXT: dup v2.4s, w81797; CHECK-NEON-NEXT: umull2 v1.2d, v0.4s, v2.4s1798; CHECK-NEON-NEXT: umull v0.2d, v0.2s, v2.2s1799; CHECK-NEON-NEXT: ret1800;1801; CHECK-SVE-LABEL: umull_and_v4i64_dup:1802; CHECK-SVE: // %bb.0: // %entry1803; CHECK-SVE-NEXT: and w8, w0, #0xff1804; CHECK-SVE-NEXT: dup v2.4s, w81805; CHECK-SVE-NEXT: umull2 v1.2d, v0.4s, v2.4s1806; CHECK-SVE-NEXT: umull v0.2d, v0.2s, v2.2s1807; CHECK-SVE-NEXT: ret1808;1809; CHECK-GI-LABEL: umull_and_v4i64_dup:1810; CHECK-GI: // %bb.0: // %entry1811; CHECK-GI-NEXT: and x8, x0, #0xff1812; CHECK-GI-NEXT: mov d2, v0.d[1]1813; CHECK-GI-NEXT: dup v1.2d, x81814; CHECK-GI-NEXT: xtn v1.2s, v1.2d1815; CHECK-GI-NEXT: umull v0.2d, v0.2s, v1.2s1816; CHECK-GI-NEXT: umull v1.2d, v2.2s, v1.2s1817; CHECK-GI-NEXT: ret1818entry:1819 %in1 = zext <4 x i32> %src1 to <4 x i64>1820 %in2 = and i64 %src2, 2551821 %broadcast.splatinsert = insertelement <4 x i64> undef, i64 %in2, i64 01822 %broadcast.splat = shufflevector <4 x i64> %broadcast.splatinsert, <4 x i64> undef, <4 x i32> zeroinitializer1823 %out = mul nsw <4 x i64> %in1, %broadcast.splat1824 ret <4 x i64> %out1825}1826 1827define void @pmlsl2_v8i16_uzp1(<16 x i8> %0, <8 x i16> %1, ptr %2, ptr %3) {1828; CHECK-NEON-LABEL: pmlsl2_v8i16_uzp1:1829; CHECK-NEON: // %bb.0:1830; CHECK-NEON-NEXT: ldr q2, [x1, #16]1831; CHECK-NEON-NEXT: uzp1 v2.16b, v0.16b, v2.16b1832; CHECK-NEON-NEXT: pmull2 v0.8h, v0.16b, v2.16b1833; CHECK-NEON-NEXT: sub v0.8h, v1.8h, v0.8h1834; CHECK-NEON-NEXT: str q0, [x0]1835; CHECK-NEON-NEXT: ret1836;1837; CHECK-SVE-LABEL: pmlsl2_v8i16_uzp1:1838; CHECK-SVE: // %bb.0:1839; CHECK-SVE-NEXT: ldr q2, [x1, #16]1840; CHECK-SVE-NEXT: uzp1 v2.16b, v0.16b, v2.16b1841; CHECK-SVE-NEXT: pmull2 v0.8h, v0.16b, v2.16b1842; CHECK-SVE-NEXT: sub v0.8h, v1.8h, v0.8h1843; CHECK-SVE-NEXT: str q0, [x0]1844; CHECK-SVE-NEXT: ret1845;1846; CHECK-GI-LABEL: pmlsl2_v8i16_uzp1:1847; CHECK-GI: // %bb.0:1848; CHECK-GI-NEXT: ldr q2, [x1, #16]1849; CHECK-GI-NEXT: mov d0, v0.d[1]1850; CHECK-GI-NEXT: xtn v2.8b, v2.8h1851; CHECK-GI-NEXT: pmull v0.8h, v0.8b, v2.8b1852; CHECK-GI-NEXT: sub v0.8h, v1.8h, v0.8h1853; CHECK-GI-NEXT: str q0, [x0]1854; CHECK-GI-NEXT: ret1855 %5 = getelementptr inbounds i32, ptr %3, i64 41856 %6 = load <8 x i16>, ptr %5, align 41857 %7 = trunc <8 x i16> %6 to <8 x i8>1858 %8 = shufflevector <16 x i8> %0, <16 x i8> poison, <8 x i32> <i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>1859 %9 = tail call <8 x i16> @llvm.aarch64.neon.pmull.v8i16(<8 x i8> %8, <8 x i8> %7)1860 %10 = sub <8 x i16> %1, %91861 store <8 x i16> %10, ptr %2, align 161862 ret void1863}1864 1865define void @smlsl2_v8i16_uzp1(<16 x i8> %0, <8 x i16> %1, ptr %2, ptr %3) {1866; CHECK-NEON-LABEL: smlsl2_v8i16_uzp1:1867; CHECK-NEON: // %bb.0:1868; CHECK-NEON-NEXT: ldr q2, [x1, #16]1869; CHECK-NEON-NEXT: uzp1 v2.16b, v0.16b, v2.16b1870; CHECK-NEON-NEXT: smlsl2 v1.8h, v0.16b, v2.16b1871; CHECK-NEON-NEXT: str q1, [x0]1872; CHECK-NEON-NEXT: ret1873;1874; CHECK-SVE-LABEL: smlsl2_v8i16_uzp1:1875; CHECK-SVE: // %bb.0:1876; CHECK-SVE-NEXT: ldr q2, [x1, #16]1877; CHECK-SVE-NEXT: uzp1 v2.16b, v0.16b, v2.16b1878; CHECK-SVE-NEXT: smlsl2 v1.8h, v0.16b, v2.16b1879; CHECK-SVE-NEXT: str q1, [x0]1880; CHECK-SVE-NEXT: ret1881;1882; CHECK-GI-LABEL: smlsl2_v8i16_uzp1:1883; CHECK-GI: // %bb.0:1884; CHECK-GI-NEXT: ldr q2, [x1, #16]1885; CHECK-GI-NEXT: mov d0, v0.d[1]1886; CHECK-GI-NEXT: xtn v2.8b, v2.8h1887; CHECK-GI-NEXT: smlsl v1.8h, v0.8b, v2.8b1888; CHECK-GI-NEXT: str q1, [x0]1889; CHECK-GI-NEXT: ret1890 %5 = getelementptr inbounds i32, ptr %3, i64 41891 %6 = load <8 x i16>, ptr %5, align 41892 %7 = trunc <8 x i16> %6 to <8 x i8>1893 %8 = shufflevector <16 x i8> %0, <16 x i8> poison, <8 x i32> <i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>1894 %9 = tail call <8 x i16> @llvm.aarch64.neon.smull.v8i16(<8 x i8> %8, <8 x i8> %7)1895 %10 = sub <8 x i16> %1, %91896 store <8 x i16> %10, ptr %2, align 161897 ret void1898}1899 1900define void @umlsl2_v8i16_uzp1(<16 x i8> %0, <8 x i16> %1, ptr %2, ptr %3) {1901; CHECK-NEON-LABEL: umlsl2_v8i16_uzp1:1902; CHECK-NEON: // %bb.0:1903; CHECK-NEON-NEXT: ldr q2, [x1, #16]1904; CHECK-NEON-NEXT: uzp1 v2.16b, v0.16b, v2.16b1905; CHECK-NEON-NEXT: umlsl2 v1.8h, v0.16b, v2.16b1906; CHECK-NEON-NEXT: str q1, [x0]1907; CHECK-NEON-NEXT: ret1908;1909; CHECK-SVE-LABEL: umlsl2_v8i16_uzp1:1910; CHECK-SVE: // %bb.0:1911; CHECK-SVE-NEXT: ldr q2, [x1, #16]1912; CHECK-SVE-NEXT: uzp1 v2.16b, v0.16b, v2.16b1913; CHECK-SVE-NEXT: umlsl2 v1.8h, v0.16b, v2.16b1914; CHECK-SVE-NEXT: str q1, [x0]1915; CHECK-SVE-NEXT: ret1916;1917; CHECK-GI-LABEL: umlsl2_v8i16_uzp1:1918; CHECK-GI: // %bb.0:1919; CHECK-GI-NEXT: ldr q2, [x1, #16]1920; CHECK-GI-NEXT: mov d0, v0.d[1]1921; CHECK-GI-NEXT: xtn v2.8b, v2.8h1922; CHECK-GI-NEXT: umlsl v1.8h, v0.8b, v2.8b1923; CHECK-GI-NEXT: str q1, [x0]1924; CHECK-GI-NEXT: ret1925 %5 = getelementptr inbounds i32, ptr %3, i64 41926 %6 = load <8 x i16>, ptr %5, align 41927 %7 = trunc <8 x i16> %6 to <8 x i8>1928 %8 = shufflevector <16 x i8> %0, <16 x i8> poison, <8 x i32> <i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>1929 %9 = tail call <8 x i16> @llvm.aarch64.neon.umull.v8i16(<8 x i8> %8, <8 x i8> %7)1930 %10 = sub <8 x i16> %1, %91931 store <8 x i16> %10, ptr %2, align 161932 ret void1933}1934 1935define void @smlsl2_v4i32_uzp1(<8 x i16> %0, <4 x i32> %1, ptr %2, ptr %3) {1936; CHECK-NEON-LABEL: smlsl2_v4i32_uzp1:1937; CHECK-NEON: // %bb.0:1938; CHECK-NEON-NEXT: ldr q2, [x1, #16]1939; CHECK-NEON-NEXT: uzp1 v2.8h, v0.8h, v2.8h1940; CHECK-NEON-NEXT: smlsl2 v1.4s, v0.8h, v2.8h1941; CHECK-NEON-NEXT: str q1, [x0]1942; CHECK-NEON-NEXT: ret1943;1944; CHECK-SVE-LABEL: smlsl2_v4i32_uzp1:1945; CHECK-SVE: // %bb.0:1946; CHECK-SVE-NEXT: ldr q2, [x1, #16]1947; CHECK-SVE-NEXT: uzp1 v2.8h, v0.8h, v2.8h1948; CHECK-SVE-NEXT: smlsl2 v1.4s, v0.8h, v2.8h1949; CHECK-SVE-NEXT: str q1, [x0]1950; CHECK-SVE-NEXT: ret1951;1952; CHECK-GI-LABEL: smlsl2_v4i32_uzp1:1953; CHECK-GI: // %bb.0:1954; CHECK-GI-NEXT: ldr q2, [x1, #16]1955; CHECK-GI-NEXT: mov d0, v0.d[1]1956; CHECK-GI-NEXT: xtn v2.4h, v2.4s1957; CHECK-GI-NEXT: smlsl v1.4s, v0.4h, v2.4h1958; CHECK-GI-NEXT: str q1, [x0]1959; CHECK-GI-NEXT: ret1960 %5 = getelementptr inbounds i32, ptr %3, i64 41961 %6 = load <4 x i32>, ptr %5, align 41962 %7 = trunc <4 x i32> %6 to <4 x i16>1963 %8 = shufflevector <8 x i16> %0, <8 x i16> poison, <4 x i32> <i32 4, i32 5, i32 6, i32 7>1964 %9 = tail call <4 x i32> @llvm.aarch64.neon.smull.v4i32(<4 x i16> %8, <4 x i16> %7)1965 %10 = sub <4 x i32> %1, %91966 store <4 x i32> %10, ptr %2, align 161967 ret void1968}1969 1970define void @umlsl2_v4i32_uzp1(<8 x i16> %0, <4 x i32> %1, ptr %2, ptr %3) {1971; CHECK-NEON-LABEL: umlsl2_v4i32_uzp1:1972; CHECK-NEON: // %bb.0:1973; CHECK-NEON-NEXT: ldr q2, [x1, #16]1974; CHECK-NEON-NEXT: uzp1 v2.8h, v0.8h, v2.8h1975; CHECK-NEON-NEXT: umlsl2 v1.4s, v0.8h, v2.8h1976; CHECK-NEON-NEXT: str q1, [x0]1977; CHECK-NEON-NEXT: ret1978;1979; CHECK-SVE-LABEL: umlsl2_v4i32_uzp1:1980; CHECK-SVE: // %bb.0:1981; CHECK-SVE-NEXT: ldr q2, [x1, #16]1982; CHECK-SVE-NEXT: uzp1 v2.8h, v0.8h, v2.8h1983; CHECK-SVE-NEXT: umlsl2 v1.4s, v0.8h, v2.8h1984; CHECK-SVE-NEXT: str q1, [x0]1985; CHECK-SVE-NEXT: ret1986;1987; CHECK-GI-LABEL: umlsl2_v4i32_uzp1:1988; CHECK-GI: // %bb.0:1989; CHECK-GI-NEXT: ldr q2, [x1, #16]1990; CHECK-GI-NEXT: mov d0, v0.d[1]1991; CHECK-GI-NEXT: xtn v2.4h, v2.4s1992; CHECK-GI-NEXT: umlsl v1.4s, v0.4h, v2.4h1993; CHECK-GI-NEXT: str q1, [x0]1994; CHECK-GI-NEXT: ret1995 %5 = getelementptr inbounds i32, ptr %3, i64 41996 %6 = load <4 x i32>, ptr %5, align 41997 %7 = trunc <4 x i32> %6 to <4 x i16>1998 %8 = shufflevector <8 x i16> %0, <8 x i16> poison, <4 x i32> <i32 4, i32 5, i32 6, i32 7>1999 %9 = tail call <4 x i32> @llvm.aarch64.neon.umull.v4i32(<4 x i16> %8, <4 x i16> %7)2000 %10 = sub <4 x i32> %1, %92001 store <4 x i32> %10, ptr %2, align 162002 ret void2003}2004 2005define void @pmlsl_pmlsl2_v8i16_uzp1(<16 x i8> %0, <8 x i16> %1, ptr %2, ptr %3, i32 %4) {2006; CHECK-NEON-LABEL: pmlsl_pmlsl2_v8i16_uzp1:2007; CHECK-NEON: // %bb.0: // %entry2008; CHECK-NEON-NEXT: ldp q2, q3, [x1]2009; CHECK-NEON-NEXT: uzp1 v2.16b, v2.16b, v3.16b2010; CHECK-NEON-NEXT: pmull v3.8h, v0.8b, v2.8b2011; CHECK-NEON-NEXT: pmull2 v0.8h, v0.16b, v2.16b2012; CHECK-NEON-NEXT: add v0.8h, v3.8h, v0.8h2013; CHECK-NEON-NEXT: sub v0.8h, v1.8h, v0.8h2014; CHECK-NEON-NEXT: str q0, [x0]2015; CHECK-NEON-NEXT: ret2016;2017; CHECK-SVE-LABEL: pmlsl_pmlsl2_v8i16_uzp1:2018; CHECK-SVE: // %bb.0: // %entry2019; CHECK-SVE-NEXT: ldp q2, q3, [x1]2020; CHECK-SVE-NEXT: uzp1 v2.16b, v2.16b, v3.16b2021; CHECK-SVE-NEXT: pmull v3.8h, v0.8b, v2.8b2022; CHECK-SVE-NEXT: pmull2 v0.8h, v0.16b, v2.16b2023; CHECK-SVE-NEXT: add v0.8h, v3.8h, v0.8h2024; CHECK-SVE-NEXT: sub v0.8h, v1.8h, v0.8h2025; CHECK-SVE-NEXT: str q0, [x0]2026; CHECK-SVE-NEXT: ret2027;2028; CHECK-GI-LABEL: pmlsl_pmlsl2_v8i16_uzp1:2029; CHECK-GI: // %bb.0: // %entry2030; CHECK-GI-NEXT: ldp q2, q3, [x1]2031; CHECK-GI-NEXT: mov d4, v0.d[1]2032; CHECK-GI-NEXT: xtn v2.8b, v2.8h2033; CHECK-GI-NEXT: xtn v3.8b, v3.8h2034; CHECK-GI-NEXT: pmull v0.8h, v0.8b, v2.8b2035; CHECK-GI-NEXT: pmull v2.8h, v4.8b, v3.8b2036; CHECK-GI-NEXT: add v0.8h, v0.8h, v2.8h2037; CHECK-GI-NEXT: sub v0.8h, v1.8h, v0.8h2038; CHECK-GI-NEXT: str q0, [x0]2039; CHECK-GI-NEXT: ret2040entry:2041 %5 = load <8 x i16>, ptr %3, align 42042 %6 = trunc <8 x i16> %5 to <8 x i8>2043 %7 = getelementptr inbounds i32, ptr %3, i64 42044 %8 = load <8 x i16>, ptr %7, align 42045 %9 = trunc <8 x i16> %8 to <8 x i8>2046 %10 = shufflevector <16 x i8> %0, <16 x i8> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>2047 %11 = tail call <8 x i16> @llvm.aarch64.neon.pmull.v8i16(<8 x i8> %10, <8 x i8> %6)2048 %12 = shufflevector <16 x i8> %0, <16 x i8> poison, <8 x i32> <i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>2049 %13 = tail call <8 x i16> @llvm.aarch64.neon.pmull.v8i16(<8 x i8> %12, <8 x i8> %9)2050 %14 = add <8 x i16> %11, %132051 %15 = sub <8 x i16> %1, %142052 store <8 x i16> %15, ptr %2, align 162053 ret void2054}2055 2056define void @smlsl_smlsl2_v8i16_uzp1(<16 x i8> %0, <8 x i16> %1, ptr %2, ptr %3, i32 %4) {2057; CHECK-NEON-LABEL: smlsl_smlsl2_v8i16_uzp1:2058; CHECK-NEON: // %bb.0: // %entry2059; CHECK-NEON-NEXT: ldp q2, q3, [x1]2060; CHECK-NEON-NEXT: uzp1 v2.16b, v2.16b, v3.16b2061; CHECK-NEON-NEXT: smlsl v1.8h, v0.8b, v2.8b2062; CHECK-NEON-NEXT: smlsl2 v1.8h, v0.16b, v2.16b2063; CHECK-NEON-NEXT: str q1, [x0]2064; CHECK-NEON-NEXT: ret2065;2066; CHECK-SVE-LABEL: smlsl_smlsl2_v8i16_uzp1:2067; CHECK-SVE: // %bb.0: // %entry2068; CHECK-SVE-NEXT: ldp q2, q3, [x1]2069; CHECK-SVE-NEXT: uzp1 v2.16b, v2.16b, v3.16b2070; CHECK-SVE-NEXT: smlsl v1.8h, v0.8b, v2.8b2071; CHECK-SVE-NEXT: smlsl2 v1.8h, v0.16b, v2.16b2072; CHECK-SVE-NEXT: str q1, [x0]2073; CHECK-SVE-NEXT: ret2074;2075; CHECK-GI-LABEL: smlsl_smlsl2_v8i16_uzp1:2076; CHECK-GI: // %bb.0: // %entry2077; CHECK-GI-NEXT: ldp q4, q2, [x1]2078; CHECK-GI-NEXT: mov d3, v0.d[1]2079; CHECK-GI-NEXT: xtn v2.8b, v2.8h2080; CHECK-GI-NEXT: xtn v4.8b, v4.8h2081; CHECK-GI-NEXT: smull v2.8h, v3.8b, v2.8b2082; CHECK-GI-NEXT: smlal v2.8h, v0.8b, v4.8b2083; CHECK-GI-NEXT: sub v0.8h, v1.8h, v2.8h2084; CHECK-GI-NEXT: str q0, [x0]2085; CHECK-GI-NEXT: ret2086entry:2087 %5 = load <8 x i16>, ptr %3, align 42088 %6 = trunc <8 x i16> %5 to <8 x i8>2089 %7 = getelementptr inbounds i32, ptr %3, i64 42090 %8 = load <8 x i16>, ptr %7, align 42091 %9 = trunc <8 x i16> %8 to <8 x i8>2092 %10 = shufflevector <16 x i8> %0, <16 x i8> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>2093 %11 = tail call <8 x i16> @llvm.aarch64.neon.smull.v8i16(<8 x i8> %10, <8 x i8> %6)2094 %12 = shufflevector <16 x i8> %0, <16 x i8> poison, <8 x i32> <i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>2095 %13 = tail call <8 x i16> @llvm.aarch64.neon.smull.v8i16(<8 x i8> %12, <8 x i8> %9)2096 %14 = add <8 x i16> %11, %132097 %15 = sub <8 x i16> %1, %142098 store <8 x i16> %15, ptr %2, align 162099 ret void2100}2101 2102define void @umlsl_umlsl2_v8i16_uzp1(<16 x i8> %0, <8 x i16> %1, ptr %2, ptr %3, i32 %4) {2103; CHECK-NEON-LABEL: umlsl_umlsl2_v8i16_uzp1:2104; CHECK-NEON: // %bb.0: // %entry2105; CHECK-NEON-NEXT: ldp q2, q3, [x1]2106; CHECK-NEON-NEXT: uzp1 v2.16b, v2.16b, v3.16b2107; CHECK-NEON-NEXT: umlsl v1.8h, v0.8b, v2.8b2108; CHECK-NEON-NEXT: umlsl2 v1.8h, v0.16b, v2.16b2109; CHECK-NEON-NEXT: str q1, [x0]2110; CHECK-NEON-NEXT: ret2111;2112; CHECK-SVE-LABEL: umlsl_umlsl2_v8i16_uzp1:2113; CHECK-SVE: // %bb.0: // %entry2114; CHECK-SVE-NEXT: ldp q2, q3, [x1]2115; CHECK-SVE-NEXT: uzp1 v2.16b, v2.16b, v3.16b2116; CHECK-SVE-NEXT: umlsl v1.8h, v0.8b, v2.8b2117; CHECK-SVE-NEXT: umlsl2 v1.8h, v0.16b, v2.16b2118; CHECK-SVE-NEXT: str q1, [x0]2119; CHECK-SVE-NEXT: ret2120;2121; CHECK-GI-LABEL: umlsl_umlsl2_v8i16_uzp1:2122; CHECK-GI: // %bb.0: // %entry2123; CHECK-GI-NEXT: ldp q4, q2, [x1]2124; CHECK-GI-NEXT: mov d3, v0.d[1]2125; CHECK-GI-NEXT: xtn v2.8b, v2.8h2126; CHECK-GI-NEXT: xtn v4.8b, v4.8h2127; CHECK-GI-NEXT: umull v2.8h, v3.8b, v2.8b2128; CHECK-GI-NEXT: umlal v2.8h, v0.8b, v4.8b2129; CHECK-GI-NEXT: sub v0.8h, v1.8h, v2.8h2130; CHECK-GI-NEXT: str q0, [x0]2131; CHECK-GI-NEXT: ret2132entry:2133 %5 = load <8 x i16>, ptr %3, align 42134 %6 = trunc <8 x i16> %5 to <8 x i8>2135 %7 = getelementptr inbounds i32, ptr %3, i64 42136 %8 = load <8 x i16>, ptr %7, align 42137 %9 = trunc <8 x i16> %8 to <8 x i8>2138 %10 = shufflevector <16 x i8> %0, <16 x i8> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>2139 %11 = tail call <8 x i16> @llvm.aarch64.neon.umull.v8i16(<8 x i8> %10, <8 x i8> %6)2140 %12 = shufflevector <16 x i8> %0, <16 x i8> poison, <8 x i32> <i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>2141 %13 = tail call <8 x i16> @llvm.aarch64.neon.umull.v8i16(<8 x i8> %12, <8 x i8> %9)2142 %14 = add <8 x i16> %11, %132143 %15 = sub <8 x i16> %1, %142144 store <8 x i16> %15, ptr %2, align 162145 ret void2146}2147 2148define void @smlsl_smlsl2_v4i32_uzp1(<8 x i16> %0, <4 x i32> %1, ptr %2, ptr %3, i32 %4) {2149; CHECK-NEON-LABEL: smlsl_smlsl2_v4i32_uzp1:2150; CHECK-NEON: // %bb.0: // %entry2151; CHECK-NEON-NEXT: ldp q2, q3, [x1]2152; CHECK-NEON-NEXT: uzp1 v2.8h, v2.8h, v3.8h2153; CHECK-NEON-NEXT: smlsl v1.4s, v0.4h, v2.4h2154; CHECK-NEON-NEXT: smlsl2 v1.4s, v0.8h, v2.8h2155; CHECK-NEON-NEXT: str q1, [x0]2156; CHECK-NEON-NEXT: ret2157;2158; CHECK-SVE-LABEL: smlsl_smlsl2_v4i32_uzp1:2159; CHECK-SVE: // %bb.0: // %entry2160; CHECK-SVE-NEXT: ldp q2, q3, [x1]2161; CHECK-SVE-NEXT: uzp1 v2.8h, v2.8h, v3.8h2162; CHECK-SVE-NEXT: smlsl v1.4s, v0.4h, v2.4h2163; CHECK-SVE-NEXT: smlsl2 v1.4s, v0.8h, v2.8h2164; CHECK-SVE-NEXT: str q1, [x0]2165; CHECK-SVE-NEXT: ret2166;2167; CHECK-GI-LABEL: smlsl_smlsl2_v4i32_uzp1:2168; CHECK-GI: // %bb.0: // %entry2169; CHECK-GI-NEXT: ldp q4, q2, [x1]2170; CHECK-GI-NEXT: mov d3, v0.d[1]2171; CHECK-GI-NEXT: xtn v2.4h, v2.4s2172; CHECK-GI-NEXT: xtn v4.4h, v4.4s2173; CHECK-GI-NEXT: smull v2.4s, v3.4h, v2.4h2174; CHECK-GI-NEXT: smlal v2.4s, v0.4h, v4.4h2175; CHECK-GI-NEXT: sub v0.4s, v1.4s, v2.4s2176; CHECK-GI-NEXT: str q0, [x0]2177; CHECK-GI-NEXT: ret2178entry:2179 %5 = load <4 x i32>, ptr %3, align 42180 %6 = trunc <4 x i32> %5 to <4 x i16>2181 %7 = getelementptr inbounds i32, ptr %3, i64 42182 %8 = load <4 x i32>, ptr %7, align 42183 %9 = trunc <4 x i32> %8 to <4 x i16>2184 %10 = shufflevector <8 x i16> %0, <8 x i16> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>2185 %11 = tail call <4 x i32> @llvm.aarch64.neon.smull.v4i32(<4 x i16> %10, <4 x i16> %6)2186 %12 = shufflevector <8 x i16> %0, <8 x i16> poison, <4 x i32> <i32 4, i32 5, i32 6, i32 7>2187 %13 = tail call <4 x i32> @llvm.aarch64.neon.smull.v4i32(<4 x i16> %12, <4 x i16> %9)2188 %14 = add <4 x i32> %11, %132189 %15 = sub <4 x i32> %1, %142190 store <4 x i32> %15, ptr %2, align 162191 ret void2192}2193 2194define void @umlsl_umlsl2_v4i32_uzp1(<8 x i16> %0, <4 x i32> %1, ptr %2, ptr %3, i32 %4) {2195; CHECK-NEON-LABEL: umlsl_umlsl2_v4i32_uzp1:2196; CHECK-NEON: // %bb.0: // %entry2197; CHECK-NEON-NEXT: ldp q2, q3, [x1]2198; CHECK-NEON-NEXT: uzp1 v2.8h, v2.8h, v3.8h2199; CHECK-NEON-NEXT: umlsl v1.4s, v0.4h, v2.4h2200; CHECK-NEON-NEXT: umlsl2 v1.4s, v0.8h, v2.8h2201; CHECK-NEON-NEXT: str q1, [x0]2202; CHECK-NEON-NEXT: ret2203;2204; CHECK-SVE-LABEL: umlsl_umlsl2_v4i32_uzp1:2205; CHECK-SVE: // %bb.0: // %entry2206; CHECK-SVE-NEXT: ldp q2, q3, [x1]2207; CHECK-SVE-NEXT: uzp1 v2.8h, v2.8h, v3.8h2208; CHECK-SVE-NEXT: umlsl v1.4s, v0.4h, v2.4h2209; CHECK-SVE-NEXT: umlsl2 v1.4s, v0.8h, v2.8h2210; CHECK-SVE-NEXT: str q1, [x0]2211; CHECK-SVE-NEXT: ret2212;2213; CHECK-GI-LABEL: umlsl_umlsl2_v4i32_uzp1:2214; CHECK-GI: // %bb.0: // %entry2215; CHECK-GI-NEXT: ldp q4, q2, [x1]2216; CHECK-GI-NEXT: mov d3, v0.d[1]2217; CHECK-GI-NEXT: xtn v2.4h, v2.4s2218; CHECK-GI-NEXT: xtn v4.4h, v4.4s2219; CHECK-GI-NEXT: umull v2.4s, v3.4h, v2.4h2220; CHECK-GI-NEXT: umlal v2.4s, v0.4h, v4.4h2221; CHECK-GI-NEXT: sub v0.4s, v1.4s, v2.4s2222; CHECK-GI-NEXT: str q0, [x0]2223; CHECK-GI-NEXT: ret2224entry:2225 %5 = load <4 x i32>, ptr %3, align 42226 %6 = trunc <4 x i32> %5 to <4 x i16>2227 %7 = getelementptr inbounds i32, ptr %3, i64 42228 %8 = load <4 x i32>, ptr %7, align 42229 %9 = trunc <4 x i32> %8 to <4 x i16>2230 %10 = shufflevector <8 x i16> %0, <8 x i16> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>2231 %11 = tail call <4 x i32> @llvm.aarch64.neon.umull.v4i32(<4 x i16> %10, <4 x i16> %6)2232 %12 = shufflevector <8 x i16> %0, <8 x i16> poison, <4 x i32> <i32 4, i32 5, i32 6, i32 7>2233 %13 = tail call <4 x i32> @llvm.aarch64.neon.umull.v4i32(<4 x i16> %12, <4 x i16> %9)2234 %14 = add <4 x i32> %11, %132235 %15 = sub <4 x i32> %1, %142236 store <4 x i32> %15, ptr %2, align 162237 ret void2238}2239 2240define <2 x i32> @do_stuff(<2 x i64> %0, <2 x i64> %1) {2241; CHECK-NEON-LABEL: do_stuff:2242; CHECK-NEON: // %bb.0:2243; CHECK-NEON-NEXT: uzp1 v0.4s, v0.4s, v0.4s2244; CHECK-NEON-NEXT: smull2 v0.2d, v1.4s, v0.4s2245; CHECK-NEON-NEXT: xtn v0.2s, v0.2d2246; CHECK-NEON-NEXT: add v0.2s, v0.2s, v1.2s2247; CHECK-NEON-NEXT: ret2248;2249; CHECK-SVE-LABEL: do_stuff:2250; CHECK-SVE: // %bb.0:2251; CHECK-SVE-NEXT: uzp1 v0.4s, v0.4s, v0.4s2252; CHECK-SVE-NEXT: smull2 v0.2d, v1.4s, v0.4s2253; CHECK-SVE-NEXT: xtn v0.2s, v0.2d2254; CHECK-SVE-NEXT: add v0.2s, v0.2s, v1.2s2255; CHECK-SVE-NEXT: ret2256;2257; CHECK-GI-LABEL: do_stuff:2258; CHECK-GI: // %bb.0:2259; CHECK-GI-NEXT: xtn v0.2s, v0.2d2260; CHECK-GI-NEXT: mov d2, v1.d[1]2261; CHECK-GI-NEXT: smull v0.2d, v2.2s, v0.2s2262; CHECK-GI-NEXT: xtn v0.2s, v0.2d2263; CHECK-GI-NEXT: add v0.2s, v0.2s, v1.2s2264; CHECK-GI-NEXT: ret2265 %bc.1 = bitcast <2 x i64> %1 to <4 x i32>2266 %trunc.0 = trunc <2 x i64> %0 to <2 x i32>2267 %shuff.hi = shufflevector <4 x i32> %bc.1, <4 x i32> zeroinitializer, <2 x i32> <i32 2, i32 3>2268 %shuff.lo = shufflevector <4 x i32> %bc.1, <4 x i32> zeroinitializer, <2 x i32> <i32 0, i32 1>2269 %smull = tail call <2 x i64> @llvm.aarch64.neon.smull.v2i64(<2 x i32> %shuff.hi, <2 x i32> %trunc.0)2270 %trunc.smull = trunc <2 x i64> %smull to <2 x i32>2271 %final = add <2 x i32> %trunc.smull, %shuff.lo2272 ret <2 x i32> %final2273}2274 2275define <2 x i64> @lsr(<2 x i64> %a, <2 x i64> %b) {2276; CHECK-LABEL: lsr:2277; CHECK: // %bb.0:2278; CHECK-NEXT: shrn v0.2s, v0.2d, #322279; CHECK-NEXT: shrn v1.2s, v1.2d, #322280; CHECK-NEXT: umull v0.2d, v0.2s, v1.2s2281; CHECK-NEXT: ret2282 %x = lshr <2 x i64> %a, <i64 32, i64 32>2283 %y = lshr <2 x i64> %b, <i64 32, i64 32>2284 %z = mul nsw <2 x i64> %x, %y2285 ret <2 x i64> %z2286}2287 2288define <2 x i64> @lsr_const(<2 x i64> %a, <2 x i64> %b) {2289; CHECK-LABEL: lsr_const:2290; CHECK: // %bb.0:2291; CHECK-NEXT: movi v1.2s, #312292; CHECK-NEXT: shrn v0.2s, v0.2d, #322293; CHECK-NEXT: umull v0.2d, v0.2s, v1.2s2294; CHECK-NEXT: ret2295 %x = lshr <2 x i64> %a, <i64 32, i64 32>2296 %z = mul nsw <2 x i64> %x, <i64 31, i64 31>2297 ret <2 x i64> %z2298}2299 2300define <2 x i64> @asr(<2 x i64> %a, <2 x i64> %b) {2301; CHECK-LABEL: asr:2302; CHECK: // %bb.0:2303; CHECK-NEXT: shrn v0.2s, v0.2d, #322304; CHECK-NEXT: shrn v1.2s, v1.2d, #322305; CHECK-NEXT: smull v0.2d, v0.2s, v1.2s2306; CHECK-NEXT: ret2307 %x = ashr <2 x i64> %a, <i64 32, i64 32>2308 %y = ashr <2 x i64> %b, <i64 32, i64 32>2309 %z = mul nsw <2 x i64> %x, %y2310 ret <2 x i64> %z2311}2312 2313define <2 x i64> @asr_const(<2 x i64> %a, <2 x i64> %b) {2314; CHECK-LABEL: asr_const:2315; CHECK: // %bb.0:2316; CHECK-NEXT: movi v1.2s, #312317; CHECK-NEXT: shrn v0.2s, v0.2d, #322318; CHECK-NEXT: smull v0.2d, v0.2s, v1.2s2319; CHECK-NEXT: ret2320 %x = ashr <2 x i64> %a, <i64 32, i64 32>2321 %z = mul nsw <2 x i64> %x, <i64 31, i64 31>2322 ret <2 x i64> %z2323}2324 2325define <8 x i16> @smulladdl_v8i8_v8i16(<8 x i8> %A, <8 x i8> %B, <8 x i8> %C) nounwind {2326; CHECK-NEON-LABEL: smulladdl_v8i8_v8i16:2327; CHECK-NEON: // %bb.0:2328; CHECK-NEON-NEXT: smull v0.8h, v0.8b, v1.8b2329; CHECK-NEON-NEXT: saddw v0.8h, v0.8h, v2.8b2330; CHECK-NEON-NEXT: ret2331;2332; CHECK-SVE-LABEL: smulladdl_v8i8_v8i16:2333; CHECK-SVE: // %bb.0:2334; CHECK-SVE-NEXT: smull v0.8h, v0.8b, v1.8b2335; CHECK-SVE-NEXT: saddw v0.8h, v0.8h, v2.8b2336; CHECK-SVE-NEXT: ret2337;2338; CHECK-GI-LABEL: smulladdl_v8i8_v8i16:2339; CHECK-GI: // %bb.0:2340; CHECK-GI-NEXT: sshll v2.8h, v2.8b, #02341; CHECK-GI-NEXT: smlal v2.8h, v0.8b, v1.8b2342; CHECK-GI-NEXT: mov v0.16b, v2.16b2343; CHECK-GI-NEXT: ret2344 %tmp1 = sext <8 x i8> %A to <8 x i16>2345 %tmp2 = sext <8 x i8> %B to <8 x i16>2346 %tmp3 = sext <8 x i8> %C to <8 x i16>2347 %tmp4 = mul <8 x i16> %tmp1, %tmp22348 %tmp5 = add <8 x i16> %tmp4, %tmp32349 ret <8 x i16> %tmp52350}2351 2352define <8 x i16> @umulladdl_v8i8_v8i16(<8 x i8> %A, <8 x i8> %B, <8 x i8> %C) nounwind {2353; CHECK-NEON-LABEL: umulladdl_v8i8_v8i16:2354; CHECK-NEON: // %bb.0:2355; CHECK-NEON-NEXT: umull v0.8h, v0.8b, v1.8b2356; CHECK-NEON-NEXT: uaddw v0.8h, v0.8h, v2.8b2357; CHECK-NEON-NEXT: ret2358;2359; CHECK-SVE-LABEL: umulladdl_v8i8_v8i16:2360; CHECK-SVE: // %bb.0:2361; CHECK-SVE-NEXT: umull v0.8h, v0.8b, v1.8b2362; CHECK-SVE-NEXT: uaddw v0.8h, v0.8h, v2.8b2363; CHECK-SVE-NEXT: ret2364;2365; CHECK-GI-LABEL: umulladdl_v8i8_v8i16:2366; CHECK-GI: // %bb.0:2367; CHECK-GI-NEXT: ushll v2.8h, v2.8b, #02368; CHECK-GI-NEXT: umlal v2.8h, v0.8b, v1.8b2369; CHECK-GI-NEXT: mov v0.16b, v2.16b2370; CHECK-GI-NEXT: ret2371 %tmp1 = zext <8 x i8> %A to <8 x i16>2372 %tmp2 = zext <8 x i8> %B to <8 x i16>2373 %tmp3 = zext <8 x i8> %C to <8 x i16>2374 %tmp4 = mul <8 x i16> %tmp1, %tmp22375 %tmp5 = add <8 x i16> %tmp4, %tmp32376 ret <8 x i16> %tmp52377}2378 2379define <8 x i16> @smlall_v8i8_v8i16(<8 x i8> %A, <8 x i8> %B, <8 x i16> %C) nounwind {2380; CHECK-LABEL: smlall_v8i8_v8i16:2381; CHECK: // %bb.0:2382; CHECK-NEXT: smlal v2.8h, v0.8b, v1.8b2383; CHECK-NEXT: mov v0.16b, v2.16b2384; CHECK-NEXT: ret2385 %tmp1 = sext <8 x i8> %A to <8 x i16>2386 %tmp2 = sext <8 x i8> %B to <8 x i16>2387 %tmp4 = mul <8 x i16> %tmp1, %tmp22388 %tmp5 = add <8 x i16> %tmp4, %C2389 ret <8 x i16> %tmp52390}2391 2392define <8 x i16> @umlall_v8i8_v8i16(<8 x i8> %A, <8 x i8> %B, <8 x i16> %C) nounwind {2393; CHECK-LABEL: umlall_v8i8_v8i16:2394; CHECK: // %bb.0:2395; CHECK-NEXT: umlal v2.8h, v0.8b, v1.8b2396; CHECK-NEXT: mov v0.16b, v2.16b2397; CHECK-NEXT: ret2398 %tmp1 = zext <8 x i8> %A to <8 x i16>2399 %tmp2 = zext <8 x i8> %B to <8 x i16>2400 %tmp4 = mul <8 x i16> %tmp1, %tmp22401 %tmp5 = add <8 x i16> %tmp4, %C2402 ret <8 x i16> %tmp52403}2404 2405define <8 x i16> @smulladdl_const_v8i8_v8i16(<8 x i8> %A, <8 x i8> %C) nounwind {2406; CHECK-NEON-LABEL: smulladdl_const_v8i8_v8i16:2407; CHECK-NEON: // %bb.0:2408; CHECK-NEON-NEXT: movi v2.8b, #102409; CHECK-NEON-NEXT: smull v0.8h, v0.8b, v2.8b2410; CHECK-NEON-NEXT: saddw v0.8h, v0.8h, v1.8b2411; CHECK-NEON-NEXT: ret2412;2413; CHECK-SVE-LABEL: smulladdl_const_v8i8_v8i16:2414; CHECK-SVE: // %bb.0:2415; CHECK-SVE-NEXT: movi v2.8b, #102416; CHECK-SVE-NEXT: smull v0.8h, v0.8b, v2.8b2417; CHECK-SVE-NEXT: saddw v0.8h, v0.8h, v1.8b2418; CHECK-SVE-NEXT: ret2419;2420; CHECK-GI-LABEL: smulladdl_const_v8i8_v8i16:2421; CHECK-GI: // %bb.0:2422; CHECK-GI-NEXT: movi v2.8b, #102423; CHECK-GI-NEXT: sshll v1.8h, v1.8b, #02424; CHECK-GI-NEXT: smlal v1.8h, v0.8b, v2.8b2425; CHECK-GI-NEXT: mov v0.16b, v1.16b2426; CHECK-GI-NEXT: ret2427 %tmp1 = sext <8 x i8> %A to <8 x i16>2428 %tmp3 = sext <8 x i8> %C to <8 x i16>2429 %tmp4 = mul <8 x i16> %tmp1, <i16 10, i16 10, i16 10, i16 10, i16 10, i16 10, i16 10, i16 10>2430 %tmp5 = add <8 x i16> %tmp4, %tmp32431 ret <8 x i16> %tmp52432}2433 2434define <8 x i16> @umulladdl_const_v8i8_v8i16(<8 x i8> %A, <8 x i8> %C) nounwind {2435; CHECK-NEON-LABEL: umulladdl_const_v8i8_v8i16:2436; CHECK-NEON: // %bb.0:2437; CHECK-NEON-NEXT: movi v2.8b, #102438; CHECK-NEON-NEXT: umull v0.8h, v0.8b, v2.8b2439; CHECK-NEON-NEXT: uaddw v0.8h, v0.8h, v1.8b2440; CHECK-NEON-NEXT: ret2441;2442; CHECK-SVE-LABEL: umulladdl_const_v8i8_v8i16:2443; CHECK-SVE: // %bb.0:2444; CHECK-SVE-NEXT: movi v2.8b, #102445; CHECK-SVE-NEXT: umull v0.8h, v0.8b, v2.8b2446; CHECK-SVE-NEXT: uaddw v0.8h, v0.8h, v1.8b2447; CHECK-SVE-NEXT: ret2448;2449; CHECK-GI-LABEL: umulladdl_const_v8i8_v8i16:2450; CHECK-GI: // %bb.0:2451; CHECK-GI-NEXT: movi v2.8b, #102452; CHECK-GI-NEXT: ushll v1.8h, v1.8b, #02453; CHECK-GI-NEXT: umlal v1.8h, v0.8b, v2.8b2454; CHECK-GI-NEXT: mov v0.16b, v1.16b2455; CHECK-GI-NEXT: ret2456 %tmp1 = zext <8 x i8> %A to <8 x i16>2457 %tmp3 = zext <8 x i8> %C to <8 x i16>2458 %tmp4 = mul <8 x i16> %tmp1, <i16 10, i16 10, i16 10, i16 10, i16 10, i16 10, i16 10, i16 10>2459 %tmp5 = add <8 x i16> %tmp4, %tmp32460 ret <8 x i16> %tmp52461}2462 2463define <8 x i16> @sdistribute_v8i8(<8 x i8> %src1, <8 x i8> %src2, <8 x i8> %mul) {2464; CHECK-NEON-LABEL: sdistribute_v8i8:2465; CHECK-NEON: // %bb.0: // %entry2466; CHECK-NEON-NEXT: smull v0.8h, v0.8b, v2.8b2467; CHECK-NEON-NEXT: smlal v0.8h, v1.8b, v2.8b2468; CHECK-NEON-NEXT: ret2469;2470; CHECK-SVE-LABEL: sdistribute_v8i8:2471; CHECK-SVE: // %bb.0: // %entry2472; CHECK-SVE-NEXT: smull v0.8h, v0.8b, v2.8b2473; CHECK-SVE-NEXT: smlal v0.8h, v1.8b, v2.8b2474; CHECK-SVE-NEXT: ret2475;2476; CHECK-GI-LABEL: sdistribute_v8i8:2477; CHECK-GI: // %bb.0: // %entry2478; CHECK-GI-NEXT: sshll v2.8h, v2.8b, #02479; CHECK-GI-NEXT: saddl v0.8h, v0.8b, v1.8b2480; CHECK-GI-NEXT: mul v0.8h, v0.8h, v2.8h2481; CHECK-GI-NEXT: ret2482entry:2483 %4 = sext <8 x i8> %src1 to <8 x i16>2484 %5 = sext <8 x i8> %mul to <8 x i16>2485 %7 = sext <8 x i8> %src2 to <8 x i16>2486 %8 = add nuw nsw <8 x i16> %4, %72487 %9 = mul <8 x i16> %8, %52488 ret <8 x i16> %92489}2490 2491define <8 x i16> @sdistribute_const1_v8i8(<8 x i8> %src1, <8 x i8> %mul) {2492; CHECK-NEON-LABEL: sdistribute_const1_v8i8:2493; CHECK-NEON: // %bb.0: // %entry2494; CHECK-NEON-NEXT: movi v2.8b, #102495; CHECK-NEON-NEXT: smull v0.8h, v0.8b, v1.8b2496; CHECK-NEON-NEXT: smlal v0.8h, v2.8b, v1.8b2497; CHECK-NEON-NEXT: ret2498;2499; CHECK-SVE-LABEL: sdistribute_const1_v8i8:2500; CHECK-SVE: // %bb.0: // %entry2501; CHECK-SVE-NEXT: movi v2.8b, #102502; CHECK-SVE-NEXT: smull v0.8h, v0.8b, v1.8b2503; CHECK-SVE-NEXT: smlal v0.8h, v2.8b, v1.8b2504; CHECK-SVE-NEXT: ret2505;2506; CHECK-GI-LABEL: sdistribute_const1_v8i8:2507; CHECK-GI: // %bb.0: // %entry2508; CHECK-GI-NEXT: movi v2.8h, #102509; CHECK-GI-NEXT: sshll v1.8h, v1.8b, #02510; CHECK-GI-NEXT: saddw v0.8h, v2.8h, v0.8b2511; CHECK-GI-NEXT: mul v0.8h, v0.8h, v1.8h2512; CHECK-GI-NEXT: ret2513entry:2514 %4 = sext <8 x i8> %src1 to <8 x i16>2515 %5 = sext <8 x i8> %mul to <8 x i16>2516 %8 = add nuw nsw <8 x i16> %4, <i16 10, i16 10, i16 10, i16 10, i16 10, i16 10, i16 10, i16 10>2517 %9 = mul <8 x i16> %8, %52518 ret <8 x i16> %92519}2520 2521define <8 x i16> @sdistribute_const2_v8i8(<8 x i8> %src1, <8 x i8> %src2) {2522; CHECK-NEON-LABEL: sdistribute_const2_v8i8:2523; CHECK-NEON: // %bb.0: // %entry2524; CHECK-NEON-NEXT: movi v2.8b, #102525; CHECK-NEON-NEXT: smull v0.8h, v0.8b, v2.8b2526; CHECK-NEON-NEXT: smlal v0.8h, v1.8b, v2.8b2527; CHECK-NEON-NEXT: ret2528;2529; CHECK-SVE-LABEL: sdistribute_const2_v8i8:2530; CHECK-SVE: // %bb.0: // %entry2531; CHECK-SVE-NEXT: movi v2.8b, #102532; CHECK-SVE-NEXT: smull v0.8h, v0.8b, v2.8b2533; CHECK-SVE-NEXT: smlal v0.8h, v1.8b, v2.8b2534; CHECK-SVE-NEXT: ret2535;2536; CHECK-GI-LABEL: sdistribute_const2_v8i8:2537; CHECK-GI: // %bb.0: // %entry2538; CHECK-GI-NEXT: movi v2.8h, #102539; CHECK-GI-NEXT: saddl v0.8h, v0.8b, v1.8b2540; CHECK-GI-NEXT: mul v0.8h, v0.8h, v2.8h2541; CHECK-GI-NEXT: ret2542entry:2543 %4 = sext <8 x i8> %src1 to <8 x i16>2544 %5 = sext <8 x i8> %src2 to <8 x i16>2545 %8 = add nuw nsw <8 x i16> %4, %52546 %9 = mul <8 x i16> %8, <i16 10, i16 10, i16 10, i16 10, i16 10, i16 10, i16 10, i16 10>2547 ret <8 x i16> %92548}2549 2550define <8 x i16> @udistribute_v8i8(<8 x i8> %src1, <8 x i8> %src2, <8 x i8> %mul) {2551; CHECK-NEON-LABEL: udistribute_v8i8:2552; CHECK-NEON: // %bb.0: // %entry2553; CHECK-NEON-NEXT: umull v0.8h, v0.8b, v2.8b2554; CHECK-NEON-NEXT: umlal v0.8h, v1.8b, v2.8b2555; CHECK-NEON-NEXT: ret2556;2557; CHECK-SVE-LABEL: udistribute_v8i8:2558; CHECK-SVE: // %bb.0: // %entry2559; CHECK-SVE-NEXT: umull v0.8h, v0.8b, v2.8b2560; CHECK-SVE-NEXT: umlal v0.8h, v1.8b, v2.8b2561; CHECK-SVE-NEXT: ret2562;2563; CHECK-GI-LABEL: udistribute_v8i8:2564; CHECK-GI: // %bb.0: // %entry2565; CHECK-GI-NEXT: ushll v2.8h, v2.8b, #02566; CHECK-GI-NEXT: uaddl v0.8h, v0.8b, v1.8b2567; CHECK-GI-NEXT: mul v0.8h, v0.8h, v2.8h2568; CHECK-GI-NEXT: ret2569entry:2570 %4 = zext <8 x i8> %src1 to <8 x i16>2571 %5 = zext <8 x i8> %mul to <8 x i16>2572 %7 = zext <8 x i8> %src2 to <8 x i16>2573 %8 = add nuw nsw <8 x i16> %4, %72574 %9 = mul <8 x i16> %8, %52575 ret <8 x i16> %92576}2577 2578define <8 x i16> @udistribute_const1_v8i8(<8 x i8> %src1, <8 x i8> %mul) {2579; CHECK-NEON-LABEL: udistribute_const1_v8i8:2580; CHECK-NEON: // %bb.0: // %entry2581; CHECK-NEON-NEXT: movi v2.8b, #102582; CHECK-NEON-NEXT: umull v0.8h, v0.8b, v1.8b2583; CHECK-NEON-NEXT: umlal v0.8h, v2.8b, v1.8b2584; CHECK-NEON-NEXT: ret2585;2586; CHECK-SVE-LABEL: udistribute_const1_v8i8:2587; CHECK-SVE: // %bb.0: // %entry2588; CHECK-SVE-NEXT: movi v2.8b, #102589; CHECK-SVE-NEXT: umull v0.8h, v0.8b, v1.8b2590; CHECK-SVE-NEXT: umlal v0.8h, v2.8b, v1.8b2591; CHECK-SVE-NEXT: ret2592;2593; CHECK-GI-LABEL: udistribute_const1_v8i8:2594; CHECK-GI: // %bb.0: // %entry2595; CHECK-GI-NEXT: movi v2.8h, #102596; CHECK-GI-NEXT: ushll v1.8h, v1.8b, #02597; CHECK-GI-NEXT: uaddw v0.8h, v2.8h, v0.8b2598; CHECK-GI-NEXT: mul v0.8h, v0.8h, v1.8h2599; CHECK-GI-NEXT: ret2600entry:2601 %4 = zext <8 x i8> %src1 to <8 x i16>2602 %5 = zext <8 x i8> %mul to <8 x i16>2603 %8 = add nuw nsw <8 x i16> %4, <i16 10, i16 10, i16 10, i16 10, i16 10, i16 10, i16 10, i16 10>2604 %9 = mul <8 x i16> %8, %52605 ret <8 x i16> %92606}2607 2608define <8 x i16> @udistribute_const2_v8i8(<8 x i8> %src1, <8 x i8> %src2) {2609; CHECK-NEON-LABEL: udistribute_const2_v8i8:2610; CHECK-NEON: // %bb.0: // %entry2611; CHECK-NEON-NEXT: movi v2.8b, #102612; CHECK-NEON-NEXT: umull v0.8h, v0.8b, v2.8b2613; CHECK-NEON-NEXT: umlal v0.8h, v1.8b, v2.8b2614; CHECK-NEON-NEXT: ret2615;2616; CHECK-SVE-LABEL: udistribute_const2_v8i8:2617; CHECK-SVE: // %bb.0: // %entry2618; CHECK-SVE-NEXT: movi v2.8b, #102619; CHECK-SVE-NEXT: umull v0.8h, v0.8b, v2.8b2620; CHECK-SVE-NEXT: umlal v0.8h, v1.8b, v2.8b2621; CHECK-SVE-NEXT: ret2622;2623; CHECK-GI-LABEL: udistribute_const2_v8i8:2624; CHECK-GI: // %bb.0: // %entry2625; CHECK-GI-NEXT: movi v2.8h, #102626; CHECK-GI-NEXT: uaddl v0.8h, v0.8b, v1.8b2627; CHECK-GI-NEXT: mul v0.8h, v0.8h, v2.8h2628; CHECK-GI-NEXT: ret2629entry:2630 %4 = zext <8 x i8> %src1 to <8 x i16>2631 %5 = zext <8 x i8> %src2 to <8 x i16>2632 %8 = add nuw nsw <8 x i16> %4, %52633 %9 = mul <8 x i16> %8, <i16 10, i16 10, i16 10, i16 10, i16 10, i16 10, i16 10, i16 10>2634 ret <8 x i16> %92635}2636 2637 2638define <2 x i64> @smulladdl_v2i32_v2i64(<2 x i32> %A, <2 x i32> %B, <2 x i32> %C) nounwind {2639; CHECK-NEON-LABEL: smulladdl_v2i32_v2i64:2640; CHECK-NEON: // %bb.0:2641; CHECK-NEON-NEXT: smull v0.2d, v0.2s, v1.2s2642; CHECK-NEON-NEXT: saddw v0.2d, v0.2d, v2.2s2643; CHECK-NEON-NEXT: ret2644;2645; CHECK-SVE-LABEL: smulladdl_v2i32_v2i64:2646; CHECK-SVE: // %bb.0:2647; CHECK-SVE-NEXT: smull v0.2d, v0.2s, v1.2s2648; CHECK-SVE-NEXT: saddw v0.2d, v0.2d, v2.2s2649; CHECK-SVE-NEXT: ret2650;2651; CHECK-GI-LABEL: smulladdl_v2i32_v2i64:2652; CHECK-GI: // %bb.0:2653; CHECK-GI-NEXT: sshll v2.2d, v2.2s, #02654; CHECK-GI-NEXT: smlal v2.2d, v0.2s, v1.2s2655; CHECK-GI-NEXT: mov v0.16b, v2.16b2656; CHECK-GI-NEXT: ret2657 %tmp1 = sext <2 x i32> %A to <2 x i64>2658 %tmp2 = sext <2 x i32> %B to <2 x i64>2659 %tmp3 = sext <2 x i32> %C to <2 x i64>2660 %tmp4 = mul <2 x i64> %tmp1, %tmp22661 %tmp5 = add <2 x i64> %tmp4, %tmp32662 ret <2 x i64> %tmp52663}2664 2665define <2 x i64> @umulladdl_v2i32_v2i64(<2 x i32> %A, <2 x i32> %B, <2 x i32> %C) nounwind {2666; CHECK-NEON-LABEL: umulladdl_v2i32_v2i64:2667; CHECK-NEON: // %bb.0:2668; CHECK-NEON-NEXT: umull v0.2d, v0.2s, v1.2s2669; CHECK-NEON-NEXT: uaddw v0.2d, v0.2d, v2.2s2670; CHECK-NEON-NEXT: ret2671;2672; CHECK-SVE-LABEL: umulladdl_v2i32_v2i64:2673; CHECK-SVE: // %bb.0:2674; CHECK-SVE-NEXT: umull v0.2d, v0.2s, v1.2s2675; CHECK-SVE-NEXT: uaddw v0.2d, v0.2d, v2.2s2676; CHECK-SVE-NEXT: ret2677;2678; CHECK-GI-LABEL: umulladdl_v2i32_v2i64:2679; CHECK-GI: // %bb.0:2680; CHECK-GI-NEXT: ushll v2.2d, v2.2s, #02681; CHECK-GI-NEXT: umlal v2.2d, v0.2s, v1.2s2682; CHECK-GI-NEXT: mov v0.16b, v2.16b2683; CHECK-GI-NEXT: ret2684 %tmp1 = zext <2 x i32> %A to <2 x i64>2685 %tmp2 = zext <2 x i32> %B to <2 x i64>2686 %tmp3 = zext <2 x i32> %C to <2 x i64>2687 %tmp4 = mul <2 x i64> %tmp1, %tmp22688 %tmp5 = add <2 x i64> %tmp4, %tmp32689 ret <2 x i64> %tmp52690}2691 2692define <2 x i64> @smlall_v2i32_v2i64(<2 x i32> %A, <2 x i32> %B, <2 x i64> %C) nounwind {2693; CHECK-LABEL: smlall_v2i32_v2i64:2694; CHECK: // %bb.0:2695; CHECK-NEXT: smlal v2.2d, v0.2s, v1.2s2696; CHECK-NEXT: mov v0.16b, v2.16b2697; CHECK-NEXT: ret2698 %tmp1 = sext <2 x i32> %A to <2 x i64>2699 %tmp2 = sext <2 x i32> %B to <2 x i64>2700 %tmp4 = mul <2 x i64> %tmp1, %tmp22701 %tmp5 = add <2 x i64> %tmp4, %C2702 ret <2 x i64> %tmp52703}2704 2705define <2 x i64> @umlall_v2i32_v2i64(<2 x i32> %A, <2 x i32> %B, <2 x i64> %C) nounwind {2706; CHECK-LABEL: umlall_v2i32_v2i64:2707; CHECK: // %bb.0:2708; CHECK-NEXT: umlal v2.2d, v0.2s, v1.2s2709; CHECK-NEXT: mov v0.16b, v2.16b2710; CHECK-NEXT: ret2711 %tmp1 = zext <2 x i32> %A to <2 x i64>2712 %tmp2 = zext <2 x i32> %B to <2 x i64>2713 %tmp4 = mul <2 x i64> %tmp1, %tmp22714 %tmp5 = add <2 x i64> %tmp4, %C2715 ret <2 x i64> %tmp52716}2717 2718define <2 x i64> @smulladdl_const_v2i32_v2i64(<2 x i32> %A, <2 x i32> %C) nounwind {2719; CHECK-NEON-LABEL: smulladdl_const_v2i32_v2i64:2720; CHECK-NEON: // %bb.0:2721; CHECK-NEON-NEXT: movi v2.2s, #102722; CHECK-NEON-NEXT: smull v0.2d, v0.2s, v2.2s2723; CHECK-NEON-NEXT: saddw v0.2d, v0.2d, v1.2s2724; CHECK-NEON-NEXT: ret2725;2726; CHECK-SVE-LABEL: smulladdl_const_v2i32_v2i64:2727; CHECK-SVE: // %bb.0:2728; CHECK-SVE-NEXT: movi v2.2s, #102729; CHECK-SVE-NEXT: smull v0.2d, v0.2s, v2.2s2730; CHECK-SVE-NEXT: saddw v0.2d, v0.2d, v1.2s2731; CHECK-SVE-NEXT: ret2732;2733; CHECK-GI-LABEL: smulladdl_const_v2i32_v2i64:2734; CHECK-GI: // %bb.0:2735; CHECK-GI-NEXT: movi v2.2s, #102736; CHECK-GI-NEXT: sshll v1.2d, v1.2s, #02737; CHECK-GI-NEXT: smlal v1.2d, v0.2s, v2.2s2738; CHECK-GI-NEXT: mov v0.16b, v1.16b2739; CHECK-GI-NEXT: ret2740 %tmp1 = sext <2 x i32> %A to <2 x i64>2741 %tmp3 = sext <2 x i32> %C to <2 x i64>2742 %tmp4 = mul <2 x i64> %tmp1, <i64 10, i64 10>2743 %tmp5 = add <2 x i64> %tmp4, %tmp32744 ret <2 x i64> %tmp52745}2746 2747define <2 x i64> @umulladdl_const_v2i32_v2i64(<2 x i32> %A, <2 x i32> %C) nounwind {2748; CHECK-NEON-LABEL: umulladdl_const_v2i32_v2i64:2749; CHECK-NEON: // %bb.0:2750; CHECK-NEON-NEXT: movi v2.2s, #102751; CHECK-NEON-NEXT: umull v0.2d, v0.2s, v2.2s2752; CHECK-NEON-NEXT: uaddw v0.2d, v0.2d, v1.2s2753; CHECK-NEON-NEXT: ret2754;2755; CHECK-SVE-LABEL: umulladdl_const_v2i32_v2i64:2756; CHECK-SVE: // %bb.0:2757; CHECK-SVE-NEXT: movi v2.2s, #102758; CHECK-SVE-NEXT: umull v0.2d, v0.2s, v2.2s2759; CHECK-SVE-NEXT: uaddw v0.2d, v0.2d, v1.2s2760; CHECK-SVE-NEXT: ret2761;2762; CHECK-GI-LABEL: umulladdl_const_v2i32_v2i64:2763; CHECK-GI: // %bb.0:2764; CHECK-GI-NEXT: movi v2.2s, #102765; CHECK-GI-NEXT: ushll v1.2d, v1.2s, #02766; CHECK-GI-NEXT: umlal v1.2d, v0.2s, v2.2s2767; CHECK-GI-NEXT: mov v0.16b, v1.16b2768; CHECK-GI-NEXT: ret2769 %tmp1 = zext <2 x i32> %A to <2 x i64>2770 %tmp3 = zext <2 x i32> %C to <2 x i64>2771 %tmp4 = mul <2 x i64> %tmp1, <i64 10, i64 10>2772 %tmp5 = add <2 x i64> %tmp4, %tmp32773 ret <2 x i64> %tmp52774}2775 2776define <2 x i64> @sdistribute_v2i32(<2 x i32> %src1, <2 x i32> %src2, <2 x i32> %mul) {2777; CHECK-NEON-LABEL: sdistribute_v2i32:2778; CHECK-NEON: // %bb.0: // %entry2779; CHECK-NEON-NEXT: smull v0.2d, v0.2s, v2.2s2780; CHECK-NEON-NEXT: smlal v0.2d, v1.2s, v2.2s2781; CHECK-NEON-NEXT: ret2782;2783; CHECK-SVE-LABEL: sdistribute_v2i32:2784; CHECK-SVE: // %bb.0: // %entry2785; CHECK-SVE-NEXT: smull v0.2d, v0.2s, v2.2s2786; CHECK-SVE-NEXT: smlal v0.2d, v1.2s, v2.2s2787; CHECK-SVE-NEXT: ret2788;2789; CHECK-GI-LABEL: sdistribute_v2i32:2790; CHECK-GI: // %bb.0: // %entry2791; CHECK-GI-NEXT: sshll v2.2d, v2.2s, #02792; CHECK-GI-NEXT: saddl v0.2d, v0.2s, v1.2s2793; CHECK-GI-NEXT: fmov x10, d02794; CHECK-GI-NEXT: fmov x11, d22795; CHECK-GI-NEXT: mov x8, v0.d[1]2796; CHECK-GI-NEXT: mov x9, v2.d[1]2797; CHECK-GI-NEXT: mul x10, x10, x112798; CHECK-GI-NEXT: mul x8, x8, x92799; CHECK-GI-NEXT: fmov d0, x102800; CHECK-GI-NEXT: mov v0.d[1], x82801; CHECK-GI-NEXT: ret2802entry:2803 %4 = sext <2 x i32> %src1 to <2 x i64>2804 %5 = sext <2 x i32> %mul to <2 x i64>2805 %7 = sext <2 x i32> %src2 to <2 x i64>2806 %8 = add nuw nsw <2 x i64> %4, %72807 %9 = mul <2 x i64> %8, %52808 ret <2 x i64> %92809}2810 2811define <2 x i64> @sdistribute_const1_v2i32(<2 x i32> %src1, <2 x i32> %mul) {2812; CHECK-NEON-LABEL: sdistribute_const1_v2i32:2813; CHECK-NEON: // %bb.0: // %entry2814; CHECK-NEON-NEXT: movi v2.2s, #102815; CHECK-NEON-NEXT: smull v0.2d, v0.2s, v1.2s2816; CHECK-NEON-NEXT: smlal v0.2d, v2.2s, v1.2s2817; CHECK-NEON-NEXT: ret2818;2819; CHECK-SVE-LABEL: sdistribute_const1_v2i32:2820; CHECK-SVE: // %bb.0: // %entry2821; CHECK-SVE-NEXT: movi v2.2s, #102822; CHECK-SVE-NEXT: smull v0.2d, v0.2s, v1.2s2823; CHECK-SVE-NEXT: smlal v0.2d, v2.2s, v1.2s2824; CHECK-SVE-NEXT: ret2825;2826; CHECK-GI-LABEL: sdistribute_const1_v2i32:2827; CHECK-GI: // %bb.0: // %entry2828; CHECK-GI-NEXT: adrp x8, .LCPI101_02829; CHECK-GI-NEXT: sshll v1.2d, v1.2s, #02830; CHECK-GI-NEXT: ldr q2, [x8, :lo12:.LCPI101_0]2831; CHECK-GI-NEXT: saddw v0.2d, v2.2d, v0.2s2832; CHECK-GI-NEXT: fmov x11, d12833; CHECK-GI-NEXT: mov x9, v1.d[1]2834; CHECK-GI-NEXT: fmov x10, d02835; CHECK-GI-NEXT: mov x8, v0.d[1]2836; CHECK-GI-NEXT: mul x10, x10, x112837; CHECK-GI-NEXT: mul x8, x8, x92838; CHECK-GI-NEXT: fmov d0, x102839; CHECK-GI-NEXT: mov v0.d[1], x82840; CHECK-GI-NEXT: ret2841entry:2842 %4 = sext <2 x i32> %src1 to <2 x i64>2843 %5 = sext <2 x i32> %mul to <2 x i64>2844 %8 = add nuw nsw <2 x i64> %4, <i64 10, i64 10>2845 %9 = mul <2 x i64> %8, %52846 ret <2 x i64> %92847}2848 2849define <2 x i64> @sdistribute_const2_v2i32(<2 x i32> %src1, <2 x i32> %src2) {2850; CHECK-NEON-LABEL: sdistribute_const2_v2i32:2851; CHECK-NEON: // %bb.0: // %entry2852; CHECK-NEON-NEXT: movi v2.2s, #102853; CHECK-NEON-NEXT: smull v0.2d, v0.2s, v2.2s2854; CHECK-NEON-NEXT: smlal v0.2d, v1.2s, v2.2s2855; CHECK-NEON-NEXT: ret2856;2857; CHECK-SVE-LABEL: sdistribute_const2_v2i32:2858; CHECK-SVE: // %bb.0: // %entry2859; CHECK-SVE-NEXT: movi v2.2s, #102860; CHECK-SVE-NEXT: smull v0.2d, v0.2s, v2.2s2861; CHECK-SVE-NEXT: smlal v0.2d, v1.2s, v2.2s2862; CHECK-SVE-NEXT: ret2863;2864; CHECK-GI-LABEL: sdistribute_const2_v2i32:2865; CHECK-GI: // %bb.0: // %entry2866; CHECK-GI-NEXT: adrp x8, .LCPI102_02867; CHECK-GI-NEXT: saddl v0.2d, v0.2s, v1.2s2868; CHECK-GI-NEXT: ldr q1, [x8, :lo12:.LCPI102_0]2869; CHECK-GI-NEXT: fmov x10, d02870; CHECK-GI-NEXT: fmov x11, d12871; CHECK-GI-NEXT: mov x8, v0.d[1]2872; CHECK-GI-NEXT: mov x9, v1.d[1]2873; CHECK-GI-NEXT: mul x10, x10, x112874; CHECK-GI-NEXT: mul x8, x8, x92875; CHECK-GI-NEXT: fmov d0, x102876; CHECK-GI-NEXT: mov v0.d[1], x82877; CHECK-GI-NEXT: ret2878entry:2879 %4 = sext <2 x i32> %src1 to <2 x i64>2880 %5 = sext <2 x i32> %src2 to <2 x i64>2881 %8 = add nuw nsw <2 x i64> %4, %52882 %9 = mul <2 x i64> %8, <i64 10, i64 10>2883 ret <2 x i64> %92884}2885 2886define <2 x i64> @udistribute_v2i32(<2 x i32> %src1, <2 x i32> %src2, <2 x i32> %mul) {2887; CHECK-NEON-LABEL: udistribute_v2i32:2888; CHECK-NEON: // %bb.0: // %entry2889; CHECK-NEON-NEXT: umull v0.2d, v0.2s, v2.2s2890; CHECK-NEON-NEXT: umlal v0.2d, v1.2s, v2.2s2891; CHECK-NEON-NEXT: ret2892;2893; CHECK-SVE-LABEL: udistribute_v2i32:2894; CHECK-SVE: // %bb.0: // %entry2895; CHECK-SVE-NEXT: umull v0.2d, v0.2s, v2.2s2896; CHECK-SVE-NEXT: umlal v0.2d, v1.2s, v2.2s2897; CHECK-SVE-NEXT: ret2898;2899; CHECK-GI-LABEL: udistribute_v2i32:2900; CHECK-GI: // %bb.0: // %entry2901; CHECK-GI-NEXT: ushll v2.2d, v2.2s, #02902; CHECK-GI-NEXT: uaddl v0.2d, v0.2s, v1.2s2903; CHECK-GI-NEXT: fmov x10, d02904; CHECK-GI-NEXT: fmov x11, d22905; CHECK-GI-NEXT: mov x8, v0.d[1]2906; CHECK-GI-NEXT: mov x9, v2.d[1]2907; CHECK-GI-NEXT: mul x10, x10, x112908; CHECK-GI-NEXT: mul x8, x8, x92909; CHECK-GI-NEXT: fmov d0, x102910; CHECK-GI-NEXT: mov v0.d[1], x82911; CHECK-GI-NEXT: ret2912entry:2913 %4 = zext <2 x i32> %src1 to <2 x i64>2914 %5 = zext <2 x i32> %mul to <2 x i64>2915 %7 = zext <2 x i32> %src2 to <2 x i64>2916 %8 = add nuw nsw <2 x i64> %4, %72917 %9 = mul <2 x i64> %8, %52918 ret <2 x i64> %92919}2920 2921define <2 x i64> @udistribute_const1_v2i32(<2 x i32> %src1, <2 x i32> %mul) {2922; CHECK-NEON-LABEL: udistribute_const1_v2i32:2923; CHECK-NEON: // %bb.0: // %entry2924; CHECK-NEON-NEXT: movi v2.2s, #102925; CHECK-NEON-NEXT: umull v0.2d, v0.2s, v1.2s2926; CHECK-NEON-NEXT: umlal v0.2d, v2.2s, v1.2s2927; CHECK-NEON-NEXT: ret2928;2929; CHECK-SVE-LABEL: udistribute_const1_v2i32:2930; CHECK-SVE: // %bb.0: // %entry2931; CHECK-SVE-NEXT: movi v2.2s, #102932; CHECK-SVE-NEXT: umull v0.2d, v0.2s, v1.2s2933; CHECK-SVE-NEXT: umlal v0.2d, v2.2s, v1.2s2934; CHECK-SVE-NEXT: ret2935;2936; CHECK-GI-LABEL: udistribute_const1_v2i32:2937; CHECK-GI: // %bb.0: // %entry2938; CHECK-GI-NEXT: adrp x8, .LCPI104_02939; CHECK-GI-NEXT: ushll v1.2d, v1.2s, #02940; CHECK-GI-NEXT: ldr q2, [x8, :lo12:.LCPI104_0]2941; CHECK-GI-NEXT: uaddw v0.2d, v2.2d, v0.2s2942; CHECK-GI-NEXT: fmov x11, d12943; CHECK-GI-NEXT: mov x9, v1.d[1]2944; CHECK-GI-NEXT: fmov x10, d02945; CHECK-GI-NEXT: mov x8, v0.d[1]2946; CHECK-GI-NEXT: mul x10, x10, x112947; CHECK-GI-NEXT: mul x8, x8, x92948; CHECK-GI-NEXT: fmov d0, x102949; CHECK-GI-NEXT: mov v0.d[1], x82950; CHECK-GI-NEXT: ret2951entry:2952 %4 = zext <2 x i32> %src1 to <2 x i64>2953 %5 = zext <2 x i32> %mul to <2 x i64>2954 %8 = add nuw nsw <2 x i64> %4, <i64 10, i64 10>2955 %9 = mul <2 x i64> %8, %52956 ret <2 x i64> %92957}2958 2959define <2 x i64> @udistribute_const2_v2i32(<2 x i32> %src1, <2 x i32> %src2) {2960; CHECK-NEON-LABEL: udistribute_const2_v2i32:2961; CHECK-NEON: // %bb.0: // %entry2962; CHECK-NEON-NEXT: movi v2.2s, #102963; CHECK-NEON-NEXT: umull v0.2d, v0.2s, v2.2s2964; CHECK-NEON-NEXT: umlal v0.2d, v1.2s, v2.2s2965; CHECK-NEON-NEXT: ret2966;2967; CHECK-SVE-LABEL: udistribute_const2_v2i32:2968; CHECK-SVE: // %bb.0: // %entry2969; CHECK-SVE-NEXT: movi v2.2s, #102970; CHECK-SVE-NEXT: umull v0.2d, v0.2s, v2.2s2971; CHECK-SVE-NEXT: umlal v0.2d, v1.2s, v2.2s2972; CHECK-SVE-NEXT: ret2973;2974; CHECK-GI-LABEL: udistribute_const2_v2i32:2975; CHECK-GI: // %bb.0: // %entry2976; CHECK-GI-NEXT: adrp x8, .LCPI105_02977; CHECK-GI-NEXT: uaddl v0.2d, v0.2s, v1.2s2978; CHECK-GI-NEXT: ldr q1, [x8, :lo12:.LCPI105_0]2979; CHECK-GI-NEXT: fmov x10, d02980; CHECK-GI-NEXT: fmov x11, d12981; CHECK-GI-NEXT: mov x8, v0.d[1]2982; CHECK-GI-NEXT: mov x9, v1.d[1]2983; CHECK-GI-NEXT: mul x10, x10, x112984; CHECK-GI-NEXT: mul x8, x8, x92985; CHECK-GI-NEXT: fmov d0, x102986; CHECK-GI-NEXT: mov v0.d[1], x82987; CHECK-GI-NEXT: ret2988entry:2989 %4 = zext <2 x i32> %src1 to <2 x i64>2990 %5 = zext <2 x i32> %src2 to <2 x i64>2991 %8 = add nuw nsw <2 x i64> %4, %52992 %9 = mul <2 x i64> %8, <i64 10, i64 10>2993 ret <2 x i64> %92994}2995 2996declare <8 x i16> @llvm.aarch64.neon.pmull.v8i16(<8 x i8>, <8 x i8>)2997declare <8 x i16> @llvm.aarch64.neon.smull.v8i16(<8 x i8>, <8 x i8>)2998declare <8 x i16> @llvm.aarch64.neon.umull.v8i16(<8 x i8>, <8 x i8>)2999declare <4 x i32> @llvm.aarch64.neon.smull.v4i32(<4 x i16>, <4 x i16>)3000declare <4 x i32> @llvm.aarch64.neon.umull.v4i32(<4 x i16>, <4 x i16>)3001declare <2 x i64> @llvm.aarch64.neon.smull.v2i64(<2 x i32>, <2 x i32>)3002