795 lines · plain
1; NOTE: Assertions have been autogenerated by utils/update_test_checks.py2; RUN: opt -S -mtriple=amdgcn-- -mcpu=tahiti -passes=amdgpu-codegenprepare %s | FileCheck -check-prefix=SI %s3; RUN: opt -S -mtriple=amdgcn-- -mcpu=fiji -amdgpu-codegenprepare %s | FileCheck -check-prefix=VI %s4; RUN: opt -S -mtriple=amdgcn-- -mcpu=fiji -amdgpu-codegenprepare-mul24=0 -amdgpu-codegenprepare %s | FileCheck -check-prefix=DISABLED %s5 6define i16 @mul_i16(i16 %lhs, i16 %rhs) {7; SI-LABEL: @mul_i16(8; SI-NEXT: [[TMP1:%.*]] = zext i16 [[LHS:%.*]] to i329; SI-NEXT: [[TMP2:%.*]] = zext i16 [[RHS:%.*]] to i3210; SI-NEXT: [[TMP3:%.*]] = call i32 @llvm.amdgcn.mul.u24.i32(i32 [[TMP1]], i32 [[TMP2]])11; SI-NEXT: [[MUL:%.*]] = trunc i32 [[TMP3]] to i1612; SI-NEXT: ret i16 [[MUL]]13;14; VI-LABEL: @mul_i16(15; VI-NEXT: [[MUL:%.*]] = mul i16 [[LHS:%.*]], [[RHS:%.*]]16; VI-NEXT: ret i16 [[MUL]]17;18; DISABLED-LABEL: @mul_i16(19; DISABLED-NEXT: [[MUL:%.*]] = mul i16 [[LHS:%.*]], [[RHS:%.*]]20; DISABLED-NEXT: ret i16 [[MUL]]21;22 %mul = mul i16 %lhs, %rhs23 ret i16 %mul24}25 26define i32 @smul24_i32(i32 %lhs, i32 %rhs) {27; SI-LABEL: @smul24_i32(28; SI-NEXT: [[SHL_LHS:%.*]] = shl i32 [[LHS:%.*]], 829; SI-NEXT: [[LHS24:%.*]] = ashr i32 [[SHL_LHS]], 830; SI-NEXT: [[SHL_RHS:%.*]] = shl i32 [[RHS:%.*]], 831; SI-NEXT: [[RHS24:%.*]] = ashr i32 [[SHL_RHS]], 832; SI-NEXT: [[MUL:%.*]] = call i32 @llvm.amdgcn.mul.i24.i32(i32 [[LHS24]], i32 [[RHS24]])33; SI-NEXT: ret i32 [[MUL]]34;35; VI-LABEL: @smul24_i32(36; VI-NEXT: [[SHL_LHS:%.*]] = shl i32 [[LHS:%.*]], 837; VI-NEXT: [[LHS24:%.*]] = ashr i32 [[SHL_LHS]], 838; VI-NEXT: [[SHL_RHS:%.*]] = shl i32 [[RHS:%.*]], 839; VI-NEXT: [[RHS24:%.*]] = ashr i32 [[SHL_RHS]], 840; VI-NEXT: [[MUL:%.*]] = call i32 @llvm.amdgcn.mul.i24.i32(i32 [[LHS24]], i32 [[RHS24]])41; VI-NEXT: ret i32 [[MUL]]42;43; DISABLED-LABEL: @smul24_i32(44; DISABLED-NEXT: [[SHL_LHS:%.*]] = shl i32 [[LHS:%.*]], 845; DISABLED-NEXT: [[LHS24:%.*]] = ashr i32 [[SHL_LHS]], 846; DISABLED-NEXT: [[SHL_RHS:%.*]] = shl i32 [[RHS:%.*]], 847; DISABLED-NEXT: [[RHS24:%.*]] = ashr i32 [[SHL_RHS]], 848; DISABLED-NEXT: [[MUL:%.*]] = mul i32 [[LHS24]], [[RHS24]]49; DISABLED-NEXT: ret i32 [[MUL]]50;51 %shl.lhs = shl i32 %lhs, 852 %lhs24 = ashr i32 %shl.lhs, 853 %shl.rhs = shl i32 %rhs, 854 %rhs24 = ashr i32 %shl.rhs, 855 %mul = mul i32 %lhs24, %rhs2456 ret i32 %mul57}58 59define <2 x i8> @mul_v1i16(<1 x i16> %arg) {60; SI-LABEL: @mul_v1i16(61; SI-NEXT: BB:62; SI-NEXT: [[TMP0:%.*]] = extractelement <1 x i16> [[ARG:%.*]], i64 063; SI-NEXT: [[TMP1:%.*]] = zext i16 [[TMP0]] to i3264; SI-NEXT: [[TMP2:%.*]] = call i32 @llvm.amdgcn.mul.u24.i32(i32 [[TMP1]], i32 42)65; SI-NEXT: [[TMP3:%.*]] = trunc i32 [[TMP2]] to i1666; SI-NEXT: [[MUL:%.*]] = insertelement <1 x i16> poison, i16 [[TMP3]], i64 067; SI-NEXT: [[CAST:%.*]] = bitcast <1 x i16> [[MUL]] to <2 x i8>68; SI-NEXT: ret <2 x i8> [[CAST]]69;70; VI-LABEL: @mul_v1i16(71; VI-NEXT: BB:72; VI-NEXT: [[MUL:%.*]] = mul <1 x i16> [[ARG:%.*]], splat (i16 42)73; VI-NEXT: [[CAST:%.*]] = bitcast <1 x i16> [[MUL]] to <2 x i8>74; VI-NEXT: ret <2 x i8> [[CAST]]75;76; DISABLED-LABEL: @mul_v1i16(77; DISABLED-NEXT: BB:78; DISABLED-NEXT: [[MUL:%.*]] = mul <1 x i16> [[ARG:%.*]], splat (i16 42)79; DISABLED-NEXT: [[CAST:%.*]] = bitcast <1 x i16> [[MUL]] to <2 x i8>80; DISABLED-NEXT: ret <2 x i8> [[CAST]]81;82BB:83 %mul = mul <1 x i16> %arg, <i16 42>84 %cast = bitcast <1 x i16> %mul to <2 x i8>85 ret <2 x i8> %cast86}87 88define <1 x i8> @mul_v1i8(<1 x i8> %arg) {89; SI-LABEL: @mul_v1i8(90; SI-NEXT: BB:91; SI-NEXT: [[TMP0:%.*]] = extractelement <1 x i8> [[ARG:%.*]], i64 092; SI-NEXT: [[TMP1:%.*]] = zext i8 [[TMP0]] to i3293; SI-NEXT: [[TMP2:%.*]] = call i32 @llvm.amdgcn.mul.u24.i32(i32 [[TMP1]], i32 42)94; SI-NEXT: [[TMP3:%.*]] = trunc i32 [[TMP2]] to i895; SI-NEXT: [[MUL:%.*]] = insertelement <1 x i8> poison, i8 [[TMP3]], i64 096; SI-NEXT: ret <1 x i8> [[MUL]]97;98; VI-LABEL: @mul_v1i8(99; VI-NEXT: BB:100; VI-NEXT: [[MUL:%.*]] = mul <1 x i8> [[ARG:%.*]], splat (i8 42)101; VI-NEXT: ret <1 x i8> [[MUL]]102;103; DISABLED-LABEL: @mul_v1i8(104; DISABLED-NEXT: BB:105; DISABLED-NEXT: [[MUL:%.*]] = mul <1 x i8> [[ARG:%.*]], splat (i8 42)106; DISABLED-NEXT: ret <1 x i8> [[MUL]]107;108BB:109 %mul = mul <1 x i8> %arg, <i8 42>110 ret <1 x i8> %mul111}112 113define <2 x i32> @smul24_v2i32(<2 x i32> %lhs, <2 x i32> %rhs) {114; SI-LABEL: @smul24_v2i32(115; SI-NEXT: [[SHL_LHS:%.*]] = shl <2 x i32> [[LHS:%.*]], splat (i32 8)116; SI-NEXT: [[LHS24:%.*]] = ashr <2 x i32> [[SHL_LHS]], splat (i32 8)117; SI-NEXT: [[SHL_RHS:%.*]] = shl <2 x i32> [[RHS:%.*]], splat (i32 8)118; SI-NEXT: [[RHS24:%.*]] = ashr <2 x i32> [[SHL_RHS]], splat (i32 8)119; SI-NEXT: [[TMP1:%.*]] = extractelement <2 x i32> [[LHS24]], i64 0120; SI-NEXT: [[TMP2:%.*]] = extractelement <2 x i32> [[LHS24]], i64 1121; SI-NEXT: [[TMP3:%.*]] = extractelement <2 x i32> [[RHS24]], i64 0122; SI-NEXT: [[TMP4:%.*]] = extractelement <2 x i32> [[RHS24]], i64 1123; SI-NEXT: [[TMP5:%.*]] = call i32 @llvm.amdgcn.mul.i24.i32(i32 [[TMP1]], i32 [[TMP3]])124; SI-NEXT: [[TMP6:%.*]] = call i32 @llvm.amdgcn.mul.i24.i32(i32 [[TMP2]], i32 [[TMP4]])125; SI-NEXT: [[TMP7:%.*]] = insertelement <2 x i32> poison, i32 [[TMP5]], i64 0126; SI-NEXT: [[MUL:%.*]] = insertelement <2 x i32> [[TMP7]], i32 [[TMP6]], i64 1127; SI-NEXT: ret <2 x i32> [[MUL]]128;129; VI-LABEL: @smul24_v2i32(130; VI-NEXT: [[SHL_LHS:%.*]] = shl <2 x i32> [[LHS:%.*]], splat (i32 8)131; VI-NEXT: [[LHS24:%.*]] = ashr <2 x i32> [[SHL_LHS]], splat (i32 8)132; VI-NEXT: [[SHL_RHS:%.*]] = shl <2 x i32> [[RHS:%.*]], splat (i32 8)133; VI-NEXT: [[RHS24:%.*]] = ashr <2 x i32> [[SHL_RHS]], splat (i32 8)134; VI-NEXT: [[TMP1:%.*]] = extractelement <2 x i32> [[LHS24]], i64 0135; VI-NEXT: [[TMP2:%.*]] = extractelement <2 x i32> [[LHS24]], i64 1136; VI-NEXT: [[TMP3:%.*]] = extractelement <2 x i32> [[RHS24]], i64 0137; VI-NEXT: [[TMP4:%.*]] = extractelement <2 x i32> [[RHS24]], i64 1138; VI-NEXT: [[TMP5:%.*]] = call i32 @llvm.amdgcn.mul.i24.i32(i32 [[TMP1]], i32 [[TMP3]])139; VI-NEXT: [[TMP6:%.*]] = call i32 @llvm.amdgcn.mul.i24.i32(i32 [[TMP2]], i32 [[TMP4]])140; VI-NEXT: [[TMP7:%.*]] = insertelement <2 x i32> poison, i32 [[TMP5]], i64 0141; VI-NEXT: [[MUL:%.*]] = insertelement <2 x i32> [[TMP7]], i32 [[TMP6]], i64 1142; VI-NEXT: ret <2 x i32> [[MUL]]143;144; DISABLED-LABEL: @smul24_v2i32(145; DISABLED-NEXT: [[SHL_LHS:%.*]] = shl <2 x i32> [[LHS:%.*]], splat (i32 8)146; DISABLED-NEXT: [[LHS24:%.*]] = ashr <2 x i32> [[SHL_LHS]], splat (i32 8)147; DISABLED-NEXT: [[SHL_RHS:%.*]] = shl <2 x i32> [[RHS:%.*]], splat (i32 8)148; DISABLED-NEXT: [[RHS24:%.*]] = ashr <2 x i32> [[SHL_RHS]], splat (i32 8)149; DISABLED-NEXT: [[MUL:%.*]] = mul <2 x i32> [[LHS24]], [[RHS24]]150; DISABLED-NEXT: ret <2 x i32> [[MUL]]151;152 %shl.lhs = shl <2 x i32> %lhs, <i32 8, i32 8>153 %lhs24 = ashr <2 x i32> %shl.lhs, <i32 8, i32 8>154 %shl.rhs = shl <2 x i32> %rhs, <i32 8, i32 8>155 %rhs24 = ashr <2 x i32> %shl.rhs, <i32 8, i32 8>156 %mul = mul <2 x i32> %lhs24, %rhs24157 ret <2 x i32> %mul158}159 160define i32 @umul24_i32(i32 %lhs, i32 %rhs) {161; SI-LABEL: @umul24_i32(162; SI-NEXT: [[LHS24:%.*]] = and i32 [[LHS:%.*]], 16777215163; SI-NEXT: [[RHS24:%.*]] = and i32 [[RHS:%.*]], 16777215164; SI-NEXT: [[MUL:%.*]] = call i32 @llvm.amdgcn.mul.u24.i32(i32 [[LHS24]], i32 [[RHS24]])165; SI-NEXT: ret i32 [[MUL]]166;167; VI-LABEL: @umul24_i32(168; VI-NEXT: [[LHS24:%.*]] = and i32 [[LHS:%.*]], 16777215169; VI-NEXT: [[RHS24:%.*]] = and i32 [[RHS:%.*]], 16777215170; VI-NEXT: [[MUL:%.*]] = call i32 @llvm.amdgcn.mul.u24.i32(i32 [[LHS24]], i32 [[RHS24]])171; VI-NEXT: ret i32 [[MUL]]172;173; DISABLED-LABEL: @umul24_i32(174; DISABLED-NEXT: [[LHS24:%.*]] = and i32 [[LHS:%.*]], 16777215175; DISABLED-NEXT: [[RHS24:%.*]] = and i32 [[RHS:%.*]], 16777215176; DISABLED-NEXT: [[MUL:%.*]] = mul i32 [[LHS24]], [[RHS24]]177; DISABLED-NEXT: ret i32 [[MUL]]178;179 %lhs24 = and i32 %lhs, 16777215180 %rhs24 = and i32 %rhs, 16777215181 %mul = mul i32 %lhs24, %rhs24182 ret i32 %mul183}184 185define <2 x i32> @umul24_v2i32(<2 x i32> %lhs, <2 x i32> %rhs) {186; SI-LABEL: @umul24_v2i32(187; SI-NEXT: [[LHS24:%.*]] = and <2 x i32> [[LHS:%.*]], splat (i32 16777215)188; SI-NEXT: [[RHS24:%.*]] = and <2 x i32> [[RHS:%.*]], splat (i32 16777215)189; SI-NEXT: [[TMP1:%.*]] = extractelement <2 x i32> [[LHS24]], i64 0190; SI-NEXT: [[TMP2:%.*]] = extractelement <2 x i32> [[LHS24]], i64 1191; SI-NEXT: [[TMP3:%.*]] = extractelement <2 x i32> [[RHS24]], i64 0192; SI-NEXT: [[TMP4:%.*]] = extractelement <2 x i32> [[RHS24]], i64 1193; SI-NEXT: [[TMP5:%.*]] = call i32 @llvm.amdgcn.mul.u24.i32(i32 [[TMP1]], i32 [[TMP3]])194; SI-NEXT: [[TMP6:%.*]] = call i32 @llvm.amdgcn.mul.u24.i32(i32 [[TMP2]], i32 [[TMP4]])195; SI-NEXT: [[TMP7:%.*]] = insertelement <2 x i32> poison, i32 [[TMP5]], i64 0196; SI-NEXT: [[MUL:%.*]] = insertelement <2 x i32> [[TMP7]], i32 [[TMP6]], i64 1197; SI-NEXT: ret <2 x i32> [[MUL]]198;199; VI-LABEL: @umul24_v2i32(200; VI-NEXT: [[LHS24:%.*]] = and <2 x i32> [[LHS:%.*]], splat (i32 16777215)201; VI-NEXT: [[RHS24:%.*]] = and <2 x i32> [[RHS:%.*]], splat (i32 16777215)202; VI-NEXT: [[TMP1:%.*]] = extractelement <2 x i32> [[LHS24]], i64 0203; VI-NEXT: [[TMP2:%.*]] = extractelement <2 x i32> [[LHS24]], i64 1204; VI-NEXT: [[TMP3:%.*]] = extractelement <2 x i32> [[RHS24]], i64 0205; VI-NEXT: [[TMP4:%.*]] = extractelement <2 x i32> [[RHS24]], i64 1206; VI-NEXT: [[TMP5:%.*]] = call i32 @llvm.amdgcn.mul.u24.i32(i32 [[TMP1]], i32 [[TMP3]])207; VI-NEXT: [[TMP6:%.*]] = call i32 @llvm.amdgcn.mul.u24.i32(i32 [[TMP2]], i32 [[TMP4]])208; VI-NEXT: [[TMP7:%.*]] = insertelement <2 x i32> poison, i32 [[TMP5]], i64 0209; VI-NEXT: [[MUL:%.*]] = insertelement <2 x i32> [[TMP7]], i32 [[TMP6]], i64 1210; VI-NEXT: ret <2 x i32> [[MUL]]211;212; DISABLED-LABEL: @umul24_v2i32(213; DISABLED-NEXT: [[LHS24:%.*]] = and <2 x i32> [[LHS:%.*]], splat (i32 16777215)214; DISABLED-NEXT: [[RHS24:%.*]] = and <2 x i32> [[RHS:%.*]], splat (i32 16777215)215; DISABLED-NEXT: [[MUL:%.*]] = mul <2 x i32> [[LHS24]], [[RHS24]]216; DISABLED-NEXT: ret <2 x i32> [[MUL]]217;218 %lhs24 = and <2 x i32> %lhs, <i32 16777215, i32 16777215>219 %rhs24 = and <2 x i32> %rhs, <i32 16777215, i32 16777215>220 %mul = mul <2 x i32> %lhs24, %rhs24221 ret <2 x i32> %mul222}223 224define i64 @smul24_i64(i64 %lhs, i64 %rhs) {225; SI-LABEL: @smul24_i64(226; SI-NEXT: [[SHL_LHS:%.*]] = shl i64 [[LHS:%.*]], 40227; SI-NEXT: [[LHS24:%.*]] = ashr i64 [[SHL_LHS]], 40228; SI-NEXT: [[SHL_RHS:%.*]] = shl i64 [[RHS:%.*]], 40229; SI-NEXT: [[RHS24:%.*]] = ashr i64 [[SHL_RHS]], 40230; SI-NEXT: [[TMP1:%.*]] = trunc i64 [[LHS24]] to i32231; SI-NEXT: [[TMP2:%.*]] = trunc i64 [[RHS24]] to i32232; SI-NEXT: [[MUL:%.*]] = call i64 @llvm.amdgcn.mul.i24.i64(i32 [[TMP1]], i32 [[TMP2]])233; SI-NEXT: ret i64 [[MUL]]234;235; VI-LABEL: @smul24_i64(236; VI-NEXT: [[SHL_LHS:%.*]] = shl i64 [[LHS:%.*]], 40237; VI-NEXT: [[LHS24:%.*]] = ashr i64 [[SHL_LHS]], 40238; VI-NEXT: [[SHL_RHS:%.*]] = shl i64 [[RHS:%.*]], 40239; VI-NEXT: [[RHS24:%.*]] = ashr i64 [[SHL_RHS]], 40240; VI-NEXT: [[TMP1:%.*]] = trunc i64 [[LHS24]] to i32241; VI-NEXT: [[TMP2:%.*]] = trunc i64 [[RHS24]] to i32242; VI-NEXT: [[MUL:%.*]] = call i64 @llvm.amdgcn.mul.i24.i64(i32 [[TMP1]], i32 [[TMP2]])243; VI-NEXT: ret i64 [[MUL]]244;245; DISABLED-LABEL: @smul24_i64(246; DISABLED-NEXT: [[SHL_LHS:%.*]] = shl i64 [[LHS:%.*]], 40247; DISABLED-NEXT: [[LHS24:%.*]] = ashr i64 [[SHL_LHS]], 40248; DISABLED-NEXT: [[SHL_RHS:%.*]] = shl i64 [[RHS:%.*]], 40249; DISABLED-NEXT: [[RHS24:%.*]] = ashr i64 [[SHL_RHS]], 40250; DISABLED-NEXT: [[MUL:%.*]] = mul i64 [[LHS24]], [[RHS24]]251; DISABLED-NEXT: ret i64 [[MUL]]252;253 %shl.lhs = shl i64 %lhs, 40254 %lhs24 = ashr i64 %shl.lhs, 40255 %shl.rhs = shl i64 %rhs, 40256 %rhs24 = ashr i64 %shl.rhs, 40257 %mul = mul i64 %lhs24, %rhs24258 ret i64 %mul259}260 261define i64 @smul24_i64_2(i64 %lhs, i64 %rhs) {262; SI-LABEL: @smul24_i64_2(263; SI-NEXT: [[SHL_LHS:%.*]] = shl i64 [[LHS:%.*]], 49264; SI-NEXT: [[LHS24:%.*]] = ashr i64 [[SHL_LHS]], 49265; SI-NEXT: [[SHL_RHS:%.*]] = shl i64 [[RHS:%.*]], 49266; SI-NEXT: [[RHS24:%.*]] = ashr i64 [[SHL_RHS]], 49267; SI-NEXT: [[TMP1:%.*]] = trunc i64 [[LHS24]] to i32268; SI-NEXT: [[TMP2:%.*]] = trunc i64 [[RHS24]] to i32269; SI-NEXT: [[MUL:%.*]] = call i64 @llvm.amdgcn.mul.i24.i64(i32 [[TMP1]], i32 [[TMP2]])270; SI-NEXT: ret i64 [[MUL]]271;272; VI-LABEL: @smul24_i64_2(273; VI-NEXT: [[SHL_LHS:%.*]] = shl i64 [[LHS:%.*]], 49274; VI-NEXT: [[LHS24:%.*]] = ashr i64 [[SHL_LHS]], 49275; VI-NEXT: [[SHL_RHS:%.*]] = shl i64 [[RHS:%.*]], 49276; VI-NEXT: [[RHS24:%.*]] = ashr i64 [[SHL_RHS]], 49277; VI-NEXT: [[TMP1:%.*]] = trunc i64 [[LHS24]] to i32278; VI-NEXT: [[TMP2:%.*]] = trunc i64 [[RHS24]] to i32279; VI-NEXT: [[MUL:%.*]] = call i64 @llvm.amdgcn.mul.i24.i64(i32 [[TMP1]], i32 [[TMP2]])280; VI-NEXT: ret i64 [[MUL]]281;282; DISABLED-LABEL: @smul24_i64_2(283; DISABLED-NEXT: [[SHL_LHS:%.*]] = shl i64 [[LHS:%.*]], 49284; DISABLED-NEXT: [[LHS24:%.*]] = ashr i64 [[SHL_LHS]], 49285; DISABLED-NEXT: [[SHL_RHS:%.*]] = shl i64 [[RHS:%.*]], 49286; DISABLED-NEXT: [[RHS24:%.*]] = ashr i64 [[SHL_RHS]], 49287; DISABLED-NEXT: [[MUL:%.*]] = mul i64 [[LHS24]], [[RHS24]]288; DISABLED-NEXT: ret i64 [[MUL]]289;290 %shl.lhs = shl i64 %lhs, 49291 %lhs24 = ashr i64 %shl.lhs, 49292 %shl.rhs = shl i64 %rhs, 49293 %rhs24 = ashr i64 %shl.rhs, 49294 %mul = mul i64 %lhs24, %rhs24295 ret i64 %mul296}297 298define i64 @smul24_i64_3(i64 %lhs, i64 %rhs) {299; SI-LABEL: @smul24_i64_3(300; SI-NEXT: [[LHS_TRUNC:%.*]] = trunc i64 [[LHS:%.*]] to i16301; SI-NEXT: [[LHS24:%.*]] = sext i16 [[LHS_TRUNC]] to i64302; SI-NEXT: [[RHS_TRUNC:%.*]] = trunc i64 [[RHS:%.*]] to i17303; SI-NEXT: [[RHS24:%.*]] = sext i17 [[RHS_TRUNC]] to i64304; SI-NEXT: [[TMP1:%.*]] = trunc i64 [[LHS24]] to i32305; SI-NEXT: [[TMP2:%.*]] = trunc i64 [[RHS24]] to i32306; SI-NEXT: [[MUL:%.*]] = call i64 @llvm.amdgcn.mul.i24.i64(i32 [[TMP1]], i32 [[TMP2]])307; SI-NEXT: ret i64 [[MUL]]308;309; VI-LABEL: @smul24_i64_3(310; VI-NEXT: [[LHS_TRUNC:%.*]] = trunc i64 [[LHS:%.*]] to i16311; VI-NEXT: [[LHS24:%.*]] = sext i16 [[LHS_TRUNC]] to i64312; VI-NEXT: [[RHS_TRUNC:%.*]] = trunc i64 [[RHS:%.*]] to i17313; VI-NEXT: [[RHS24:%.*]] = sext i17 [[RHS_TRUNC]] to i64314; VI-NEXT: [[TMP1:%.*]] = trunc i64 [[LHS24]] to i32315; VI-NEXT: [[TMP2:%.*]] = trunc i64 [[RHS24]] to i32316; VI-NEXT: [[MUL:%.*]] = call i64 @llvm.amdgcn.mul.i24.i64(i32 [[TMP1]], i32 [[TMP2]])317; VI-NEXT: ret i64 [[MUL]]318;319; DISABLED-LABEL: @smul24_i64_3(320; DISABLED-NEXT: [[LHS_TRUNC:%.*]] = trunc i64 [[LHS:%.*]] to i16321; DISABLED-NEXT: [[LHS24:%.*]] = sext i16 [[LHS_TRUNC]] to i64322; DISABLED-NEXT: [[RHS_TRUNC:%.*]] = trunc i64 [[RHS:%.*]] to i17323; DISABLED-NEXT: [[RHS24:%.*]] = sext i17 [[RHS_TRUNC]] to i64324; DISABLED-NEXT: [[MUL:%.*]] = mul i64 [[LHS24]], [[RHS24]]325; DISABLED-NEXT: ret i64 [[MUL]]326;327 %lhs.trunc = trunc i64 %lhs to i16328 %lhs24 = sext i16 %lhs.trunc to i64329 %rhs.trunc = trunc i64 %rhs to i17330 %rhs24 = sext i17 %rhs.trunc to i64331 %mul = mul i64 %lhs24, %rhs24332 ret i64 %mul333}334 335define i64 @smul24_i64_4(i64 %lhs, i64 %rhs) {336; SI-LABEL: @smul24_i64_4(337; SI-NEXT: [[LHS_TRUNC:%.*]] = trunc i64 [[LHS:%.*]] to i24338; SI-NEXT: [[LHS24:%.*]] = sext i24 [[LHS_TRUNC]] to i64339; SI-NEXT: [[RHS_TRUNC:%.*]] = trunc i64 [[RHS:%.*]] to i25340; SI-NEXT: [[RHS24:%.*]] = sext i25 [[RHS_TRUNC]] to i64341; SI-NEXT: [[MUL:%.*]] = mul i64 [[LHS24]], [[RHS24]]342; SI-NEXT: ret i64 [[MUL]]343;344; VI-LABEL: @smul24_i64_4(345; VI-NEXT: [[LHS_TRUNC:%.*]] = trunc i64 [[LHS:%.*]] to i24346; VI-NEXT: [[LHS24:%.*]] = sext i24 [[LHS_TRUNC]] to i64347; VI-NEXT: [[RHS_TRUNC:%.*]] = trunc i64 [[RHS:%.*]] to i25348; VI-NEXT: [[RHS24:%.*]] = sext i25 [[RHS_TRUNC]] to i64349; VI-NEXT: [[MUL:%.*]] = mul i64 [[LHS24]], [[RHS24]]350; VI-NEXT: ret i64 [[MUL]]351;352; DISABLED-LABEL: @smul24_i64_4(353; DISABLED-NEXT: [[LHS_TRUNC:%.*]] = trunc i64 [[LHS:%.*]] to i24354; DISABLED-NEXT: [[LHS24:%.*]] = sext i24 [[LHS_TRUNC]] to i64355; DISABLED-NEXT: [[RHS_TRUNC:%.*]] = trunc i64 [[RHS:%.*]] to i25356; DISABLED-NEXT: [[RHS24:%.*]] = sext i25 [[RHS_TRUNC]] to i64357; DISABLED-NEXT: [[MUL:%.*]] = mul i64 [[LHS24]], [[RHS24]]358; DISABLED-NEXT: ret i64 [[MUL]]359;360 %lhs.trunc = trunc i64 %lhs to i24361 %lhs24 = sext i24 %lhs.trunc to i64362 %rhs.trunc = trunc i64 %rhs to i25363 %rhs24 = sext i25 %rhs.trunc to i64364 %mul = mul i64 %lhs24, %rhs24365 ret i64 %mul366}367 368define i64 @umul24_i64(i64 %lhs, i64 %rhs) {369; SI-LABEL: @umul24_i64(370; SI-NEXT: [[LHS24:%.*]] = and i64 [[LHS:%.*]], 16777215371; SI-NEXT: [[RHS24:%.*]] = and i64 [[RHS:%.*]], 16777215372; SI-NEXT: [[TMP1:%.*]] = trunc i64 [[LHS24]] to i32373; SI-NEXT: [[TMP2:%.*]] = trunc i64 [[RHS24]] to i32374; SI-NEXT: [[MUL:%.*]] = call i64 @llvm.amdgcn.mul.u24.i64(i32 [[TMP1]], i32 [[TMP2]])375; SI-NEXT: ret i64 [[MUL]]376;377; VI-LABEL: @umul24_i64(378; VI-NEXT: [[LHS24:%.*]] = and i64 [[LHS:%.*]], 16777215379; VI-NEXT: [[RHS24:%.*]] = and i64 [[RHS:%.*]], 16777215380; VI-NEXT: [[TMP1:%.*]] = trunc i64 [[LHS24]] to i32381; VI-NEXT: [[TMP2:%.*]] = trunc i64 [[RHS24]] to i32382; VI-NEXT: [[MUL:%.*]] = call i64 @llvm.amdgcn.mul.u24.i64(i32 [[TMP1]], i32 [[TMP2]])383; VI-NEXT: ret i64 [[MUL]]384;385; DISABLED-LABEL: @umul24_i64(386; DISABLED-NEXT: [[LHS24:%.*]] = and i64 [[LHS:%.*]], 16777215387; DISABLED-NEXT: [[RHS24:%.*]] = and i64 [[RHS:%.*]], 16777215388; DISABLED-NEXT: [[MUL:%.*]] = mul i64 [[LHS24]], [[RHS24]]389; DISABLED-NEXT: ret i64 [[MUL]]390;391 %lhs24 = and i64 %lhs, 16777215392 %rhs24 = and i64 %rhs, 16777215393 %mul = mul i64 %lhs24, %rhs24394 ret i64 %mul395}396 397define i64 @umul24_i64_2(i64 %lhs, i64 %rhs) {398; SI-LABEL: @umul24_i64_2(399; SI-NEXT: [[LHS24:%.*]] = and i64 [[LHS:%.*]], 65535400; SI-NEXT: [[RHS24:%.*]] = and i64 [[RHS:%.*]], 65535401; SI-NEXT: [[TMP1:%.*]] = trunc i64 [[LHS24]] to i32402; SI-NEXT: [[TMP2:%.*]] = trunc i64 [[RHS24]] to i32403; SI-NEXT: [[MUL:%.*]] = call i64 @llvm.amdgcn.mul.u24.i64(i32 [[TMP1]], i32 [[TMP2]])404; SI-NEXT: ret i64 [[MUL]]405;406; VI-LABEL: @umul24_i64_2(407; VI-NEXT: [[LHS24:%.*]] = and i64 [[LHS:%.*]], 65535408; VI-NEXT: [[RHS24:%.*]] = and i64 [[RHS:%.*]], 65535409; VI-NEXT: [[TMP1:%.*]] = trunc i64 [[LHS24]] to i32410; VI-NEXT: [[TMP2:%.*]] = trunc i64 [[RHS24]] to i32411; VI-NEXT: [[MUL:%.*]] = call i64 @llvm.amdgcn.mul.u24.i64(i32 [[TMP1]], i32 [[TMP2]])412; VI-NEXT: ret i64 [[MUL]]413;414; DISABLED-LABEL: @umul24_i64_2(415; DISABLED-NEXT: [[LHS24:%.*]] = and i64 [[LHS:%.*]], 65535416; DISABLED-NEXT: [[RHS24:%.*]] = and i64 [[RHS:%.*]], 65535417; DISABLED-NEXT: [[TMP1:%.*]] = trunc i64 [[LHS24]] to i32418; DISABLED-NEXT: [[TMP2:%.*]] = trunc i64 [[RHS24]] to i32419; DISABLED-NEXT: [[TMP3:%.*]] = mul i32 [[TMP1]], [[TMP2]]420; DISABLED-NEXT: [[MUL:%.*]] = zext i32 [[TMP3]] to i64421; DISABLED-NEXT: ret i64 [[MUL]]422;423 %lhs24 = and i64 %lhs, 65535424 %rhs24 = and i64 %rhs, 65535425 %mul = mul i64 %lhs24, %rhs24426 ret i64 %mul427}428 429define i31 @smul24_i31(i31 %lhs, i31 %rhs) {430; SI-LABEL: @smul24_i31(431; SI-NEXT: [[SHL_LHS:%.*]] = shl i31 [[LHS:%.*]], 7432; SI-NEXT: [[LHS24:%.*]] = ashr i31 [[SHL_LHS]], 7433; SI-NEXT: [[SHL_RHS:%.*]] = shl i31 [[RHS:%.*]], 7434; SI-NEXT: [[RHS24:%.*]] = ashr i31 [[SHL_RHS]], 7435; SI-NEXT: [[TMP1:%.*]] = sext i31 [[LHS24]] to i32436; SI-NEXT: [[TMP2:%.*]] = sext i31 [[RHS24]] to i32437; SI-NEXT: [[TMP3:%.*]] = call i32 @llvm.amdgcn.mul.i24.i32(i32 [[TMP1]], i32 [[TMP2]])438; SI-NEXT: [[MUL:%.*]] = trunc i32 [[TMP3]] to i31439; SI-NEXT: ret i31 [[MUL]]440;441; VI-LABEL: @smul24_i31(442; VI-NEXT: [[SHL_LHS:%.*]] = shl i31 [[LHS:%.*]], 7443; VI-NEXT: [[LHS24:%.*]] = ashr i31 [[SHL_LHS]], 7444; VI-NEXT: [[SHL_RHS:%.*]] = shl i31 [[RHS:%.*]], 7445; VI-NEXT: [[RHS24:%.*]] = ashr i31 [[SHL_RHS]], 7446; VI-NEXT: [[TMP1:%.*]] = sext i31 [[LHS24]] to i32447; VI-NEXT: [[TMP2:%.*]] = sext i31 [[RHS24]] to i32448; VI-NEXT: [[TMP3:%.*]] = call i32 @llvm.amdgcn.mul.i24.i32(i32 [[TMP1]], i32 [[TMP2]])449; VI-NEXT: [[MUL:%.*]] = trunc i32 [[TMP3]] to i31450; VI-NEXT: ret i31 [[MUL]]451;452; DISABLED-LABEL: @smul24_i31(453; DISABLED-NEXT: [[SHL_LHS:%.*]] = shl i31 [[LHS:%.*]], 7454; DISABLED-NEXT: [[LHS24:%.*]] = ashr i31 [[SHL_LHS]], 7455; DISABLED-NEXT: [[SHL_RHS:%.*]] = shl i31 [[RHS:%.*]], 7456; DISABLED-NEXT: [[RHS24:%.*]] = ashr i31 [[SHL_RHS]], 7457; DISABLED-NEXT: [[MUL:%.*]] = mul i31 [[LHS24]], [[RHS24]]458; DISABLED-NEXT: ret i31 [[MUL]]459;460 %shl.lhs = shl i31 %lhs, 7461 %lhs24 = ashr i31 %shl.lhs, 7462 %shl.rhs = shl i31 %rhs, 7463 %rhs24 = ashr i31 %shl.rhs, 7464 %mul = mul i31 %lhs24, %rhs24465 ret i31 %mul466}467 468define i31 @umul24_i31(i31 %lhs, i31 %rhs) {469; SI-LABEL: @umul24_i31(470; SI-NEXT: [[LHS24:%.*]] = and i31 [[LHS:%.*]], 16777215471; SI-NEXT: [[RHS24:%.*]] = and i31 [[RHS:%.*]], 16777215472; SI-NEXT: [[TMP1:%.*]] = zext i31 [[LHS24]] to i32473; SI-NEXT: [[TMP2:%.*]] = zext i31 [[RHS24]] to i32474; SI-NEXT: [[TMP3:%.*]] = call i32 @llvm.amdgcn.mul.u24.i32(i32 [[TMP1]], i32 [[TMP2]])475; SI-NEXT: [[MUL:%.*]] = trunc i32 [[TMP3]] to i31476; SI-NEXT: ret i31 [[MUL]]477;478; VI-LABEL: @umul24_i31(479; VI-NEXT: [[LHS24:%.*]] = and i31 [[LHS:%.*]], 16777215480; VI-NEXT: [[RHS24:%.*]] = and i31 [[RHS:%.*]], 16777215481; VI-NEXT: [[TMP1:%.*]] = zext i31 [[LHS24]] to i32482; VI-NEXT: [[TMP2:%.*]] = zext i31 [[RHS24]] to i32483; VI-NEXT: [[TMP3:%.*]] = call i32 @llvm.amdgcn.mul.u24.i32(i32 [[TMP1]], i32 [[TMP2]])484; VI-NEXT: [[MUL:%.*]] = trunc i32 [[TMP3]] to i31485; VI-NEXT: ret i31 [[MUL]]486;487; DISABLED-LABEL: @umul24_i31(488; DISABLED-NEXT: [[LHS24:%.*]] = and i31 [[LHS:%.*]], 16777215489; DISABLED-NEXT: [[RHS24:%.*]] = and i31 [[RHS:%.*]], 16777215490; DISABLED-NEXT: [[MUL:%.*]] = mul i31 [[LHS24]], [[RHS24]]491; DISABLED-NEXT: ret i31 [[MUL]]492;493 %lhs24 = and i31 %lhs, 16777215494 %rhs24 = and i31 %rhs, 16777215495 %mul = mul i31 %lhs24, %rhs24496 ret i31 %mul497}498 499define <2 x i31> @umul24_v2i31(<2 x i31> %lhs, <2 x i31> %rhs) {500; SI-LABEL: @umul24_v2i31(501; SI-NEXT: [[LHS24:%.*]] = and <2 x i31> [[LHS:%.*]], splat (i31 16777215)502; SI-NEXT: [[RHS24:%.*]] = and <2 x i31> [[RHS:%.*]], splat (i31 16777215)503; SI-NEXT: [[TMP1:%.*]] = extractelement <2 x i31> [[LHS24]], i64 0504; SI-NEXT: [[TMP2:%.*]] = extractelement <2 x i31> [[LHS24]], i64 1505; SI-NEXT: [[TMP3:%.*]] = extractelement <2 x i31> [[RHS24]], i64 0506; SI-NEXT: [[TMP4:%.*]] = extractelement <2 x i31> [[RHS24]], i64 1507; SI-NEXT: [[TMP5:%.*]] = zext i31 [[TMP1]] to i32508; SI-NEXT: [[TMP6:%.*]] = zext i31 [[TMP3]] to i32509; SI-NEXT: [[TMP7:%.*]] = call i32 @llvm.amdgcn.mul.u24.i32(i32 [[TMP5]], i32 [[TMP6]])510; SI-NEXT: [[TMP8:%.*]] = trunc i32 [[TMP7]] to i31511; SI-NEXT: [[TMP9:%.*]] = zext i31 [[TMP2]] to i32512; SI-NEXT: [[TMP10:%.*]] = zext i31 [[TMP4]] to i32513; SI-NEXT: [[TMP11:%.*]] = call i32 @llvm.amdgcn.mul.u24.i32(i32 [[TMP9]], i32 [[TMP10]])514; SI-NEXT: [[TMP12:%.*]] = trunc i32 [[TMP11]] to i31515; SI-NEXT: [[TMP13:%.*]] = insertelement <2 x i31> poison, i31 [[TMP8]], i64 0516; SI-NEXT: [[MUL:%.*]] = insertelement <2 x i31> [[TMP13]], i31 [[TMP12]], i64 1517; SI-NEXT: ret <2 x i31> [[MUL]]518;519; VI-LABEL: @umul24_v2i31(520; VI-NEXT: [[LHS24:%.*]] = and <2 x i31> [[LHS:%.*]], splat (i31 16777215)521; VI-NEXT: [[RHS24:%.*]] = and <2 x i31> [[RHS:%.*]], splat (i31 16777215)522; VI-NEXT: [[TMP1:%.*]] = extractelement <2 x i31> [[LHS24]], i64 0523; VI-NEXT: [[TMP2:%.*]] = extractelement <2 x i31> [[LHS24]], i64 1524; VI-NEXT: [[TMP3:%.*]] = extractelement <2 x i31> [[RHS24]], i64 0525; VI-NEXT: [[TMP4:%.*]] = extractelement <2 x i31> [[RHS24]], i64 1526; VI-NEXT: [[TMP5:%.*]] = zext i31 [[TMP1]] to i32527; VI-NEXT: [[TMP6:%.*]] = zext i31 [[TMP3]] to i32528; VI-NEXT: [[TMP7:%.*]] = call i32 @llvm.amdgcn.mul.u24.i32(i32 [[TMP5]], i32 [[TMP6]])529; VI-NEXT: [[TMP8:%.*]] = trunc i32 [[TMP7]] to i31530; VI-NEXT: [[TMP9:%.*]] = zext i31 [[TMP2]] to i32531; VI-NEXT: [[TMP10:%.*]] = zext i31 [[TMP4]] to i32532; VI-NEXT: [[TMP11:%.*]] = call i32 @llvm.amdgcn.mul.u24.i32(i32 [[TMP9]], i32 [[TMP10]])533; VI-NEXT: [[TMP12:%.*]] = trunc i32 [[TMP11]] to i31534; VI-NEXT: [[TMP13:%.*]] = insertelement <2 x i31> poison, i31 [[TMP8]], i64 0535; VI-NEXT: [[MUL:%.*]] = insertelement <2 x i31> [[TMP13]], i31 [[TMP12]], i64 1536; VI-NEXT: ret <2 x i31> [[MUL]]537;538; DISABLED-LABEL: @umul24_v2i31(539; DISABLED-NEXT: [[LHS24:%.*]] = and <2 x i31> [[LHS:%.*]], splat (i31 16777215)540; DISABLED-NEXT: [[RHS24:%.*]] = and <2 x i31> [[RHS:%.*]], splat (i31 16777215)541; DISABLED-NEXT: [[MUL:%.*]] = mul <2 x i31> [[LHS24]], [[RHS24]]542; DISABLED-NEXT: ret <2 x i31> [[MUL]]543;544 %lhs24 = and <2 x i31> %lhs, <i31 16777215, i31 16777215>545 %rhs24 = and <2 x i31> %rhs, <i31 16777215, i31 16777215>546 %mul = mul <2 x i31> %lhs24, %rhs24547 ret <2 x i31> %mul548}549 550define <2 x i31> @smul24_v2i31(<2 x i31> %lhs, <2 x i31> %rhs) {551; SI-LABEL: @smul24_v2i31(552; SI-NEXT: [[SHL_LHS:%.*]] = shl <2 x i31> [[LHS:%.*]], splat (i31 8)553; SI-NEXT: [[LHS24:%.*]] = ashr <2 x i31> [[SHL_LHS]], splat (i31 8)554; SI-NEXT: [[SHL_RHS:%.*]] = shl <2 x i31> [[RHS:%.*]], splat (i31 8)555; SI-NEXT: [[RHS24:%.*]] = ashr <2 x i31> [[SHL_RHS]], splat (i31 8)556; SI-NEXT: [[TMP1:%.*]] = extractelement <2 x i31> [[LHS24]], i64 0557; SI-NEXT: [[TMP2:%.*]] = extractelement <2 x i31> [[LHS24]], i64 1558; SI-NEXT: [[TMP3:%.*]] = extractelement <2 x i31> [[RHS24]], i64 0559; SI-NEXT: [[TMP4:%.*]] = extractelement <2 x i31> [[RHS24]], i64 1560; SI-NEXT: [[TMP5:%.*]] = sext i31 [[TMP1]] to i32561; SI-NEXT: [[TMP6:%.*]] = sext i31 [[TMP3]] to i32562; SI-NEXT: [[TMP7:%.*]] = call i32 @llvm.amdgcn.mul.i24.i32(i32 [[TMP5]], i32 [[TMP6]])563; SI-NEXT: [[TMP8:%.*]] = trunc i32 [[TMP7]] to i31564; SI-NEXT: [[TMP9:%.*]] = sext i31 [[TMP2]] to i32565; SI-NEXT: [[TMP10:%.*]] = sext i31 [[TMP4]] to i32566; SI-NEXT: [[TMP11:%.*]] = call i32 @llvm.amdgcn.mul.i24.i32(i32 [[TMP9]], i32 [[TMP10]])567; SI-NEXT: [[TMP12:%.*]] = trunc i32 [[TMP11]] to i31568; SI-NEXT: [[TMP13:%.*]] = insertelement <2 x i31> poison, i31 [[TMP8]], i64 0569; SI-NEXT: [[MUL:%.*]] = insertelement <2 x i31> [[TMP13]], i31 [[TMP12]], i64 1570; SI-NEXT: ret <2 x i31> [[MUL]]571;572; VI-LABEL: @smul24_v2i31(573; VI-NEXT: [[SHL_LHS:%.*]] = shl <2 x i31> [[LHS:%.*]], splat (i31 8)574; VI-NEXT: [[LHS24:%.*]] = ashr <2 x i31> [[SHL_LHS]], splat (i31 8)575; VI-NEXT: [[SHL_RHS:%.*]] = shl <2 x i31> [[RHS:%.*]], splat (i31 8)576; VI-NEXT: [[RHS24:%.*]] = ashr <2 x i31> [[SHL_RHS]], splat (i31 8)577; VI-NEXT: [[TMP1:%.*]] = extractelement <2 x i31> [[LHS24]], i64 0578; VI-NEXT: [[TMP2:%.*]] = extractelement <2 x i31> [[LHS24]], i64 1579; VI-NEXT: [[TMP3:%.*]] = extractelement <2 x i31> [[RHS24]], i64 0580; VI-NEXT: [[TMP4:%.*]] = extractelement <2 x i31> [[RHS24]], i64 1581; VI-NEXT: [[TMP5:%.*]] = sext i31 [[TMP1]] to i32582; VI-NEXT: [[TMP6:%.*]] = sext i31 [[TMP3]] to i32583; VI-NEXT: [[TMP7:%.*]] = call i32 @llvm.amdgcn.mul.i24.i32(i32 [[TMP5]], i32 [[TMP6]])584; VI-NEXT: [[TMP8:%.*]] = trunc i32 [[TMP7]] to i31585; VI-NEXT: [[TMP9:%.*]] = sext i31 [[TMP2]] to i32586; VI-NEXT: [[TMP10:%.*]] = sext i31 [[TMP4]] to i32587; VI-NEXT: [[TMP11:%.*]] = call i32 @llvm.amdgcn.mul.i24.i32(i32 [[TMP9]], i32 [[TMP10]])588; VI-NEXT: [[TMP12:%.*]] = trunc i32 [[TMP11]] to i31589; VI-NEXT: [[TMP13:%.*]] = insertelement <2 x i31> poison, i31 [[TMP8]], i64 0590; VI-NEXT: [[MUL:%.*]] = insertelement <2 x i31> [[TMP13]], i31 [[TMP12]], i64 1591; VI-NEXT: ret <2 x i31> [[MUL]]592;593; DISABLED-LABEL: @smul24_v2i31(594; DISABLED-NEXT: [[SHL_LHS:%.*]] = shl <2 x i31> [[LHS:%.*]], splat (i31 8)595; DISABLED-NEXT: [[LHS24:%.*]] = ashr <2 x i31> [[SHL_LHS]], splat (i31 8)596; DISABLED-NEXT: [[SHL_RHS:%.*]] = shl <2 x i31> [[RHS:%.*]], splat (i31 8)597; DISABLED-NEXT: [[RHS24:%.*]] = ashr <2 x i31> [[SHL_RHS]], splat (i31 8)598; DISABLED-NEXT: [[MUL:%.*]] = mul <2 x i31> [[LHS24]], [[RHS24]]599; DISABLED-NEXT: ret <2 x i31> [[MUL]]600;601 %shl.lhs = shl <2 x i31> %lhs, <i31 8, i31 8>602 %lhs24 = ashr <2 x i31> %shl.lhs, <i31 8, i31 8>603 %shl.rhs = shl <2 x i31> %rhs, <i31 8, i31 8>604 %rhs24 = ashr <2 x i31> %shl.rhs, <i31 8, i31 8>605 %mul = mul <2 x i31> %lhs24, %rhs24606 ret <2 x i31> %mul607}608 609define i33 @smul24_i33(i33 %lhs, i33 %rhs) {610; SI-LABEL: @smul24_i33(611; SI-NEXT: [[SHL_LHS:%.*]] = shl i33 [[LHS:%.*]], 9612; SI-NEXT: [[LHS24:%.*]] = ashr i33 [[SHL_LHS]], 9613; SI-NEXT: [[SHL_RHS:%.*]] = shl i33 [[RHS:%.*]], 9614; SI-NEXT: [[RHS24:%.*]] = ashr i33 [[SHL_RHS]], 9615; SI-NEXT: [[TMP1:%.*]] = trunc i33 [[LHS24]] to i32616; SI-NEXT: [[TMP2:%.*]] = trunc i33 [[RHS24]] to i32617; SI-NEXT: [[TMP3:%.*]] = call i64 @llvm.amdgcn.mul.i24.i64(i32 [[TMP1]], i32 [[TMP2]])618; SI-NEXT: [[MUL:%.*]] = trunc i64 [[TMP3]] to i33619; SI-NEXT: ret i33 [[MUL]]620;621; VI-LABEL: @smul24_i33(622; VI-NEXT: [[SHL_LHS:%.*]] = shl i33 [[LHS:%.*]], 9623; VI-NEXT: [[LHS24:%.*]] = ashr i33 [[SHL_LHS]], 9624; VI-NEXT: [[SHL_RHS:%.*]] = shl i33 [[RHS:%.*]], 9625; VI-NEXT: [[RHS24:%.*]] = ashr i33 [[SHL_RHS]], 9626; VI-NEXT: [[TMP1:%.*]] = trunc i33 [[LHS24]] to i32627; VI-NEXT: [[TMP2:%.*]] = trunc i33 [[RHS24]] to i32628; VI-NEXT: [[TMP3:%.*]] = call i64 @llvm.amdgcn.mul.i24.i64(i32 [[TMP1]], i32 [[TMP2]])629; VI-NEXT: [[MUL:%.*]] = trunc i64 [[TMP3]] to i33630; VI-NEXT: ret i33 [[MUL]]631;632; DISABLED-LABEL: @smul24_i33(633; DISABLED-NEXT: [[SHL_LHS:%.*]] = shl i33 [[LHS:%.*]], 9634; DISABLED-NEXT: [[LHS24:%.*]] = ashr i33 [[SHL_LHS]], 9635; DISABLED-NEXT: [[SHL_RHS:%.*]] = shl i33 [[RHS:%.*]], 9636; DISABLED-NEXT: [[RHS24:%.*]] = ashr i33 [[SHL_RHS]], 9637; DISABLED-NEXT: [[MUL:%.*]] = mul i33 [[LHS24]], [[RHS24]]638; DISABLED-NEXT: ret i33 [[MUL]]639;640 %shl.lhs = shl i33 %lhs, 9641 %lhs24 = ashr i33 %shl.lhs, 9642 %shl.rhs = shl i33 %rhs, 9643 %rhs24 = ashr i33 %shl.rhs, 9644 %mul = mul i33 %lhs24, %rhs24645 ret i33 %mul646}647 648define i33 @umul24_i33(i33 %lhs, i33 %rhs) {649; SI-LABEL: @umul24_i33(650; SI-NEXT: [[LHS24:%.*]] = and i33 [[LHS:%.*]], 16777215651; SI-NEXT: [[RHS24:%.*]] = and i33 [[RHS:%.*]], 16777215652; SI-NEXT: [[TMP1:%.*]] = trunc i33 [[LHS24]] to i32653; SI-NEXT: [[TMP2:%.*]] = trunc i33 [[RHS24]] to i32654; SI-NEXT: [[TMP3:%.*]] = call i64 @llvm.amdgcn.mul.u24.i64(i32 [[TMP1]], i32 [[TMP2]])655; SI-NEXT: [[MUL:%.*]] = trunc i64 [[TMP3]] to i33656; SI-NEXT: ret i33 [[MUL]]657;658; VI-LABEL: @umul24_i33(659; VI-NEXT: [[LHS24:%.*]] = and i33 [[LHS:%.*]], 16777215660; VI-NEXT: [[RHS24:%.*]] = and i33 [[RHS:%.*]], 16777215661; VI-NEXT: [[TMP1:%.*]] = trunc i33 [[LHS24]] to i32662; VI-NEXT: [[TMP2:%.*]] = trunc i33 [[RHS24]] to i32663; VI-NEXT: [[TMP3:%.*]] = call i64 @llvm.amdgcn.mul.u24.i64(i32 [[TMP1]], i32 [[TMP2]])664; VI-NEXT: [[MUL:%.*]] = trunc i64 [[TMP3]] to i33665; VI-NEXT: ret i33 [[MUL]]666;667; DISABLED-LABEL: @umul24_i33(668; DISABLED-NEXT: [[LHS24:%.*]] = and i33 [[LHS:%.*]], 16777215669; DISABLED-NEXT: [[RHS24:%.*]] = and i33 [[RHS:%.*]], 16777215670; DISABLED-NEXT: [[MUL:%.*]] = mul i33 [[LHS24]], [[RHS24]]671; DISABLED-NEXT: ret i33 [[MUL]]672;673 %lhs24 = and i33 %lhs, 16777215674 %rhs24 = and i33 %rhs, 16777215675 %mul = mul i33 %lhs24, %rhs24676 ret i33 %mul677}678 679define i32 @smul25_i32(i32 %lhs, i32 %rhs) {680; SI-LABEL: @smul25_i32(681; SI-NEXT: [[SHL_LHS:%.*]] = shl i32 [[LHS:%.*]], 7682; SI-NEXT: [[LHS24:%.*]] = ashr i32 [[SHL_LHS]], 7683; SI-NEXT: [[SHL_RHS:%.*]] = shl i32 [[RHS:%.*]], 7684; SI-NEXT: [[RHS24:%.*]] = ashr i32 [[SHL_RHS]], 7685; SI-NEXT: [[MUL:%.*]] = mul i32 [[LHS24]], [[RHS24]]686; SI-NEXT: ret i32 [[MUL]]687;688; VI-LABEL: @smul25_i32(689; VI-NEXT: [[SHL_LHS:%.*]] = shl i32 [[LHS:%.*]], 7690; VI-NEXT: [[LHS24:%.*]] = ashr i32 [[SHL_LHS]], 7691; VI-NEXT: [[SHL_RHS:%.*]] = shl i32 [[RHS:%.*]], 7692; VI-NEXT: [[RHS24:%.*]] = ashr i32 [[SHL_RHS]], 7693; VI-NEXT: [[MUL:%.*]] = mul i32 [[LHS24]], [[RHS24]]694; VI-NEXT: ret i32 [[MUL]]695;696; DISABLED-LABEL: @smul25_i32(697; DISABLED-NEXT: [[SHL_LHS:%.*]] = shl i32 [[LHS:%.*]], 7698; DISABLED-NEXT: [[LHS24:%.*]] = ashr i32 [[SHL_LHS]], 7699; DISABLED-NEXT: [[SHL_RHS:%.*]] = shl i32 [[RHS:%.*]], 7700; DISABLED-NEXT: [[RHS24:%.*]] = ashr i32 [[SHL_RHS]], 7701; DISABLED-NEXT: [[MUL:%.*]] = mul i32 [[LHS24]], [[RHS24]]702; DISABLED-NEXT: ret i32 [[MUL]]703;704 %shl.lhs = shl i32 %lhs, 7705 %lhs24 = ashr i32 %shl.lhs, 7706 %shl.rhs = shl i32 %rhs, 7707 %rhs24 = ashr i32 %shl.rhs, 7708 %mul = mul i32 %lhs24, %rhs24709 ret i32 %mul710}711 712define i32 @umul25_i32(i32 %lhs, i32 %rhs) {713; SI-LABEL: @umul25_i32(714; SI-NEXT: [[LHS24:%.*]] = and i32 [[LHS:%.*]], 33554431715; SI-NEXT: [[RHS24:%.*]] = and i32 [[RHS:%.*]], 33554431716; SI-NEXT: [[MUL:%.*]] = mul i32 [[LHS24]], [[RHS24]]717; SI-NEXT: ret i32 [[MUL]]718;719; VI-LABEL: @umul25_i32(720; VI-NEXT: [[LHS24:%.*]] = and i32 [[LHS:%.*]], 33554431721; VI-NEXT: [[RHS24:%.*]] = and i32 [[RHS:%.*]], 33554431722; VI-NEXT: [[MUL:%.*]] = mul i32 [[LHS24]], [[RHS24]]723; VI-NEXT: ret i32 [[MUL]]724;725; DISABLED-LABEL: @umul25_i32(726; DISABLED-NEXT: [[LHS24:%.*]] = and i32 [[LHS:%.*]], 33554431727; DISABLED-NEXT: [[RHS24:%.*]] = and i32 [[RHS:%.*]], 33554431728; DISABLED-NEXT: [[MUL:%.*]] = mul i32 [[LHS24]], [[RHS24]]729; DISABLED-NEXT: ret i32 [[MUL]]730;731 %lhs24 = and i32 %lhs, 33554431732 %rhs24 = and i32 %rhs, 33554431733 %mul = mul i32 %lhs24, %rhs24734 ret i32 %mul735}736 737define <2 x i33> @smul24_v2i33(<2 x i33> %lhs, <2 x i33> %rhs) {738; SI-LABEL: @smul24_v2i33(739; SI-NEXT: [[SHL_LHS:%.*]] = shl <2 x i33> [[LHS:%.*]], splat (i33 9)740; SI-NEXT: [[LHS24:%.*]] = ashr <2 x i33> [[SHL_LHS]], splat (i33 9)741; SI-NEXT: [[SHL_RHS:%.*]] = shl <2 x i33> [[RHS:%.*]], splat (i33 9)742; SI-NEXT: [[RHS24:%.*]] = ashr <2 x i33> [[SHL_RHS]], splat (i33 9)743; SI-NEXT: [[TMP1:%.*]] = extractelement <2 x i33> [[LHS24]], i64 0744; SI-NEXT: [[TMP2:%.*]] = extractelement <2 x i33> [[LHS24]], i64 1745; SI-NEXT: [[TMP3:%.*]] = extractelement <2 x i33> [[RHS24]], i64 0746; SI-NEXT: [[TMP4:%.*]] = extractelement <2 x i33> [[RHS24]], i64 1747; SI-NEXT: [[TMP5:%.*]] = trunc i33 [[TMP1]] to i32748; SI-NEXT: [[TMP6:%.*]] = trunc i33 [[TMP3]] to i32749; SI-NEXT: [[TMP7:%.*]] = call i64 @llvm.amdgcn.mul.i24.i64(i32 [[TMP5]], i32 [[TMP6]])750; SI-NEXT: [[TMP8:%.*]] = trunc i64 [[TMP7]] to i33751; SI-NEXT: [[TMP9:%.*]] = trunc i33 [[TMP2]] to i32752; SI-NEXT: [[TMP10:%.*]] = trunc i33 [[TMP4]] to i32753; SI-NEXT: [[TMP11:%.*]] = call i64 @llvm.amdgcn.mul.i24.i64(i32 [[TMP9]], i32 [[TMP10]])754; SI-NEXT: [[TMP12:%.*]] = trunc i64 [[TMP11]] to i33755; SI-NEXT: [[TMP13:%.*]] = insertelement <2 x i33> poison, i33 [[TMP8]], i64 0756; SI-NEXT: [[MUL:%.*]] = insertelement <2 x i33> [[TMP13]], i33 [[TMP12]], i64 1757; SI-NEXT: ret <2 x i33> [[MUL]]758;759; VI-LABEL: @smul24_v2i33(760; VI-NEXT: [[SHL_LHS:%.*]] = shl <2 x i33> [[LHS:%.*]], splat (i33 9)761; VI-NEXT: [[LHS24:%.*]] = ashr <2 x i33> [[SHL_LHS]], splat (i33 9)762; VI-NEXT: [[SHL_RHS:%.*]] = shl <2 x i33> [[RHS:%.*]], splat (i33 9)763; VI-NEXT: [[RHS24:%.*]] = ashr <2 x i33> [[SHL_RHS]], splat (i33 9)764; VI-NEXT: [[TMP1:%.*]] = extractelement <2 x i33> [[LHS24]], i64 0765; VI-NEXT: [[TMP2:%.*]] = extractelement <2 x i33> [[LHS24]], i64 1766; VI-NEXT: [[TMP3:%.*]] = extractelement <2 x i33> [[RHS24]], i64 0767; VI-NEXT: [[TMP4:%.*]] = extractelement <2 x i33> [[RHS24]], i64 1768; VI-NEXT: [[TMP5:%.*]] = trunc i33 [[TMP1]] to i32769; VI-NEXT: [[TMP6:%.*]] = trunc i33 [[TMP3]] to i32770; VI-NEXT: [[TMP7:%.*]] = call i64 @llvm.amdgcn.mul.i24.i64(i32 [[TMP5]], i32 [[TMP6]])771; VI-NEXT: [[TMP8:%.*]] = trunc i64 [[TMP7]] to i33772; VI-NEXT: [[TMP9:%.*]] = trunc i33 [[TMP2]] to i32773; VI-NEXT: [[TMP10:%.*]] = trunc i33 [[TMP4]] to i32774; VI-NEXT: [[TMP11:%.*]] = call i64 @llvm.amdgcn.mul.i24.i64(i32 [[TMP9]], i32 [[TMP10]])775; VI-NEXT: [[TMP12:%.*]] = trunc i64 [[TMP11]] to i33776; VI-NEXT: [[TMP13:%.*]] = insertelement <2 x i33> poison, i33 [[TMP8]], i64 0777; VI-NEXT: [[MUL:%.*]] = insertelement <2 x i33> [[TMP13]], i33 [[TMP12]], i64 1778; VI-NEXT: ret <2 x i33> [[MUL]]779;780; DISABLED-LABEL: @smul24_v2i33(781; DISABLED-NEXT: [[SHL_LHS:%.*]] = shl <2 x i33> [[LHS:%.*]], splat (i33 9)782; DISABLED-NEXT: [[LHS24:%.*]] = ashr <2 x i33> [[SHL_LHS]], splat (i33 9)783; DISABLED-NEXT: [[SHL_RHS:%.*]] = shl <2 x i33> [[RHS:%.*]], splat (i33 9)784; DISABLED-NEXT: [[RHS24:%.*]] = ashr <2 x i33> [[SHL_RHS]], splat (i33 9)785; DISABLED-NEXT: [[MUL:%.*]] = mul <2 x i33> [[LHS24]], [[RHS24]]786; DISABLED-NEXT: ret <2 x i33> [[MUL]]787;788 %shl.lhs = shl <2 x i33> %lhs, <i33 9, i33 9>789 %lhs24 = ashr <2 x i33> %shl.lhs, <i33 9, i33 9>790 %shl.rhs = shl <2 x i33> %rhs, <i33 9, i33 9>791 %rhs24 = ashr <2 x i33> %shl.rhs, <i33 9, i33 9>792 %mul = mul <2 x i33> %lhs24, %rhs24793 ret <2 x i33> %mul794}795