686 lines · plain
1; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 52; RUN: llc -mtriple=amdgcn -mcpu=gfx900 < %s | FileCheck -enable-var-scope -check-prefixes=GFX9 %s3; RUN: llc -mtriple=amdgcn -mcpu=fiji < %s | FileCheck -enable-var-scope -check-prefixes=VI %s4 5define half @reduction_fadd_v4f16(<4 x half> %vec4) {6; GFX9-LABEL: reduction_fadd_v4f16:7; GFX9: ; %bb.0: ; %entry8; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)9; GFX9-NEXT: v_pk_add_f16 v0, v0, v110; GFX9-NEXT: v_add_f16_sdwa v0, v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_111; GFX9-NEXT: s_setpc_b64 s[30:31]12;13; VI-LABEL: reduction_fadd_v4f16:14; VI: ; %bb.0: ; %entry15; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)16; VI-NEXT: v_add_f16_sdwa v2, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_117; VI-NEXT: v_add_f16_e32 v0, v0, v118; VI-NEXT: v_add_f16_e32 v0, v0, v219; VI-NEXT: s_setpc_b64 s[30:31]20entry:21 %rdx.shuf = shufflevector <4 x half> %vec4, <4 x half> poison, <4 x i32> <i32 2, i32 3, i32 poison, i32 poison>22 %bin.rdx = fadd <4 x half> %vec4, %rdx.shuf23 %rdx.shuf1 = shufflevector <4 x half> %bin.rdx, <4 x half> poison, <4 x i32> <i32 1, i32 poison, i32 poison, i32 poison>24 %bin.rdx2 = fadd <4 x half> %bin.rdx, %rdx.shuf125 %res = extractelement <4 x half> %bin.rdx2, i32 026 ret half %res27}28 29define half @reduction_fsub_v4f16(<4 x half> %vec4) {30; GFX9-LABEL: reduction_fsub_v4f16:31; GFX9: ; %bb.0: ; %entry32; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)33; GFX9-NEXT: v_pk_add_f16 v0, v0, v1 neg_lo:[0,1] neg_hi:[0,1]34; GFX9-NEXT: v_sub_f16_sdwa v0, v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_135; GFX9-NEXT: s_setpc_b64 s[30:31]36;37; VI-LABEL: reduction_fsub_v4f16:38; VI: ; %bb.0: ; %entry39; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)40; VI-NEXT: v_sub_f16_sdwa v2, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_141; VI-NEXT: v_sub_f16_e32 v0, v0, v142; VI-NEXT: v_sub_f16_e32 v0, v0, v243; VI-NEXT: s_setpc_b64 s[30:31]44entry:45 %rdx.shuf = shufflevector <4 x half> %vec4, <4 x half> poison, <4 x i32> <i32 2, i32 3, i32 poison, i32 poison>46 %bin.rdx = fsub <4 x half> %vec4, %rdx.shuf47 %rdx.shuf1 = shufflevector <4 x half> %bin.rdx, <4 x half> poison, <4 x i32> <i32 1, i32 poison, i32 poison, i32 poison>48 %bin.rdx2 = fsub <4 x half> %bin.rdx, %rdx.shuf149 %res = extractelement <4 x half> %bin.rdx2, i32 050 ret half %res51}52 53; Make sure nsz is preserved when the operations are split.54define half @reduction_fsub_v4f16_preserve_fmf(<4 x half> %vec4) {55; GFX9-LABEL: reduction_fsub_v4f16_preserve_fmf:56; GFX9: ; %bb.0: ; %entry57; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)58; GFX9-NEXT: v_pk_add_f16 v0, v0, v1 neg_lo:[0,1] neg_hi:[0,1]59; GFX9-NEXT: v_sub_f16_sdwa v0, v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD60; GFX9-NEXT: s_setpc_b64 s[30:31]61;62; VI-LABEL: reduction_fsub_v4f16_preserve_fmf:63; VI: ; %bb.0: ; %entry64; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)65; VI-NEXT: v_sub_f16_sdwa v2, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_166; VI-NEXT: v_sub_f16_e32 v0, v1, v067; VI-NEXT: v_add_f16_e32 v0, v2, v068; VI-NEXT: s_setpc_b64 s[30:31]69entry:70 %rdx.shuf = shufflevector <4 x half> %vec4, <4 x half> poison, <4 x i32> <i32 2, i32 3, i32 poison, i32 poison>71 %bin.rdx = fsub nsz <4 x half> %vec4, %rdx.shuf72 %rdx.shuf1 = shufflevector <4 x half> %bin.rdx, <4 x half> poison, <4 x i32> <i32 1, i32 poison, i32 poison, i32 poison>73 %bin.rdx2 = fsub nsz <4 x half> %bin.rdx, %rdx.shuf174 %res = extractelement <4 x half> %bin.rdx2, i32 075 %neg.res = fsub half -0.0, %res76 ret half %neg.res77}78 79define half @reduction_fmul_half4(<4 x half> %vec4) {80; GFX9-LABEL: reduction_fmul_half4:81; GFX9: ; %bb.0: ; %entry82; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)83; GFX9-NEXT: v_pk_mul_f16 v0, v0, v184; GFX9-NEXT: v_mul_f16_sdwa v0, v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_185; GFX9-NEXT: s_setpc_b64 s[30:31]86;87; VI-LABEL: reduction_fmul_half4:88; VI: ; %bb.0: ; %entry89; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)90; VI-NEXT: v_mul_f16_sdwa v2, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_191; VI-NEXT: v_mul_f16_e32 v0, v0, v192; VI-NEXT: v_mul_f16_e32 v0, v0, v293; VI-NEXT: s_setpc_b64 s[30:31]94entry:95 %rdx.shuf = shufflevector <4 x half> %vec4, <4 x half> poison, <4 x i32> <i32 2, i32 3, i32 poison, i32 poison>96 %bin.rdx = fmul <4 x half> %vec4, %rdx.shuf97 %rdx.shuf1 = shufflevector <4 x half> %bin.rdx, <4 x half> poison, <4 x i32> <i32 1, i32 poison, i32 poison, i32 poison>98 %bin.rdx2 = fmul <4 x half> %bin.rdx, %rdx.shuf199 %res = extractelement <4 x half> %bin.rdx2, i32 0100 ret half %res101}102 103define i16 @reduction_v4i16(<4 x i16> %vec4) {104; GFX9-LABEL: reduction_v4i16:105; GFX9: ; %bb.0: ; %entry106; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)107; GFX9-NEXT: v_pk_add_u16 v0, v0, v1108; GFX9-NEXT: v_add_u16_sdwa v0, v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1109; GFX9-NEXT: s_setpc_b64 s[30:31]110;111; VI-LABEL: reduction_v4i16:112; VI: ; %bb.0: ; %entry113; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)114; VI-NEXT: v_add_u16_sdwa v2, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1115; VI-NEXT: v_add_u16_e32 v0, v0, v1116; VI-NEXT: v_add_u16_e32 v0, v0, v2117; VI-NEXT: s_setpc_b64 s[30:31]118entry:119 %rdx.shuf = shufflevector <4 x i16> %vec4, <4 x i16> poison, <4 x i32> <i32 2, i32 3, i32 poison, i32 poison>120 %bin.rdx = add <4 x i16> %vec4, %rdx.shuf121 %rdx.shuf1 = shufflevector <4 x i16> %bin.rdx, <4 x i16> poison, <4 x i32> <i32 1, i32 poison, i32 poison, i32 poison>122 %bin.rdx2 = add <4 x i16> %bin.rdx, %rdx.shuf1123 %res = extractelement <4 x i16> %bin.rdx2, i32 0124 ret i16 %res125}126 127define half @reduction_half8(<8 x half> %vec8) {128; GFX9-LABEL: reduction_half8:129; GFX9: ; %bb.0: ; %entry130; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)131; GFX9-NEXT: v_pk_add_f16 v1, v1, v3132; GFX9-NEXT: v_pk_add_f16 v0, v0, v2133; GFX9-NEXT: v_pk_add_f16 v0, v0, v1134; GFX9-NEXT: v_add_f16_sdwa v0, v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1135; GFX9-NEXT: s_setpc_b64 s[30:31]136;137; VI-LABEL: reduction_half8:138; VI: ; %bb.0: ; %entry139; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)140; VI-NEXT: v_add_f16_sdwa v4, v1, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1141; VI-NEXT: v_add_f16_sdwa v5, v0, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1142; VI-NEXT: v_add_f16_e32 v1, v1, v3143; VI-NEXT: v_add_f16_e32 v0, v0, v2144; VI-NEXT: v_add_f16_e32 v2, v5, v4145; VI-NEXT: v_add_f16_e32 v0, v0, v1146; VI-NEXT: v_add_f16_e32 v0, v0, v2147; VI-NEXT: s_setpc_b64 s[30:31]148entry:149 %rdx.shuf = shufflevector <8 x half> %vec8, <8 x half> poison, <8 x i32> <i32 4, i32 5, i32 6, i32 7, i32 poison, i32 poison, i32 poison, i32 poison>150 %bin.rdx = fadd <8 x half> %vec8, %rdx.shuf151 %rdx.shuf1 = shufflevector <8 x half> %bin.rdx, <8 x half> poison, <8 x i32> <i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>152 %bin.rdx2 = fadd <8 x half> %bin.rdx, %rdx.shuf1153 %rdx.shuf3 = shufflevector <8 x half> %bin.rdx2, <8 x half> poison, <8 x i32> <i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>154 %bin.rdx4 = fadd <8 x half> %bin.rdx2, %rdx.shuf3155 %res = extractelement <8 x half> %bin.rdx4, i32 0156 ret half %res157}158 159define i16 @reduction_v8i16(<8 x i16> %vec8) {160; GFX9-LABEL: reduction_v8i16:161; GFX9: ; %bb.0: ; %entry162; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)163; GFX9-NEXT: v_pk_add_u16 v1, v1, v3164; GFX9-NEXT: v_pk_add_u16 v0, v0, v2165; GFX9-NEXT: v_pk_add_u16 v0, v0, v1166; GFX9-NEXT: v_add_u16_sdwa v0, v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1167; GFX9-NEXT: s_setpc_b64 s[30:31]168;169; VI-LABEL: reduction_v8i16:170; VI: ; %bb.0: ; %entry171; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)172; VI-NEXT: v_add_u16_sdwa v4, v1, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1173; VI-NEXT: v_add_u16_sdwa v5, v0, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1174; VI-NEXT: v_add_u16_e32 v1, v1, v3175; VI-NEXT: v_add_u16_e32 v0, v0, v2176; VI-NEXT: v_add_u16_e32 v2, v5, v4177; VI-NEXT: v_add_u16_e32 v0, v0, v1178; VI-NEXT: v_add_u16_e32 v0, v0, v2179; VI-NEXT: s_setpc_b64 s[30:31]180entry:181 %rdx.shuf = shufflevector <8 x i16> %vec8, <8 x i16> poison, <8 x i32> <i32 4, i32 5, i32 6, i32 7, i32 poison, i32 poison, i32 poison, i32 poison>182 %bin.rdx = add <8 x i16> %vec8, %rdx.shuf183 %rdx.shuf1 = shufflevector <8 x i16> %bin.rdx, <8 x i16> poison, <8 x i32> <i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>184 %bin.rdx2 = add <8 x i16> %bin.rdx, %rdx.shuf1185 %rdx.shuf3 = shufflevector <8 x i16> %bin.rdx2, <8 x i16> poison, <8 x i32> <i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>186 %bin.rdx4 = add <8 x i16> %bin.rdx2, %rdx.shuf3187 %res = extractelement <8 x i16> %bin.rdx4, i32 0188 ret i16 %res189}190 191define half @reduction_half16(<16 x half> %vec16) {192; GFX9-LABEL: reduction_half16:193; GFX9: ; %bb.0: ; %entry194; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)195; GFX9-NEXT: v_pk_add_f16 v2, v2, v6196; GFX9-NEXT: v_pk_add_f16 v0, v0, v4197; GFX9-NEXT: v_pk_add_f16 v3, v3, v7198; GFX9-NEXT: v_pk_add_f16 v1, v1, v5199; GFX9-NEXT: v_pk_add_f16 v1, v1, v3200; GFX9-NEXT: v_pk_add_f16 v0, v0, v2201; GFX9-NEXT: v_pk_add_f16 v0, v0, v1202; GFX9-NEXT: v_add_f16_sdwa v0, v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1203; GFX9-NEXT: s_setpc_b64 s[30:31]204;205; VI-LABEL: reduction_half16:206; VI: ; %bb.0: ; %entry207; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)208; VI-NEXT: v_add_f16_sdwa v8, v2, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1209; VI-NEXT: v_add_f16_sdwa v9, v0, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1210; VI-NEXT: v_add_f16_sdwa v10, v3, v7 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1211; VI-NEXT: v_add_f16_sdwa v11, v1, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1212; VI-NEXT: v_add_f16_e32 v2, v2, v6213; VI-NEXT: v_add_f16_e32 v0, v0, v4214; VI-NEXT: v_add_f16_e32 v3, v3, v7215; VI-NEXT: v_add_f16_e32 v1, v1, v5216; VI-NEXT: v_add_f16_e32 v4, v11, v10217; VI-NEXT: v_add_f16_e32 v5, v9, v8218; VI-NEXT: v_add_f16_e32 v1, v1, v3219; VI-NEXT: v_add_f16_e32 v0, v0, v2220; VI-NEXT: v_add_f16_e32 v2, v5, v4221; VI-NEXT: v_add_f16_e32 v0, v0, v1222; VI-NEXT: v_add_f16_e32 v0, v0, v2223; VI-NEXT: s_setpc_b64 s[30:31]224entry:225 %rdx.shuf = shufflevector <16 x half> %vec16, <16 x half> poison, <16 x i32> <i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>226 %bin.rdx = fadd <16 x half> %vec16, %rdx.shuf227 %rdx.shuf1 = shufflevector <16 x half> %bin.rdx, <16 x half> poison, <16 x i32> <i32 4, i32 5, i32 6, i32 7, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>228 %bin.rdx2 = fadd <16 x half> %bin.rdx, %rdx.shuf1229 %rdx.shuf3 = shufflevector <16 x half> %bin.rdx2, <16 x half> poison, <16 x i32> <i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>230 %bin.rdx4 = fadd <16 x half> %bin.rdx2, %rdx.shuf3231 %rdx.shuf5 = shufflevector <16 x half> %bin.rdx4, <16 x half> poison, <16 x i32> <i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>232 %bin.rdx6 = fadd <16 x half> %bin.rdx4, %rdx.shuf5233 %res = extractelement <16 x half> %bin.rdx6, i32 0234 ret half %res235}236 237define i16 @reduction_min_v4i16(<4 x i16> %vec4) {238; GFX9-LABEL: reduction_min_v4i16:239; GFX9: ; %bb.0: ; %entry240; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)241; GFX9-NEXT: v_pk_min_u16 v0, v0, v1242; GFX9-NEXT: v_min_u16_sdwa v0, v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1243; GFX9-NEXT: s_setpc_b64 s[30:31]244;245; VI-LABEL: reduction_min_v4i16:246; VI: ; %bb.0: ; %entry247; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)248; VI-NEXT: v_min_u16_sdwa v2, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1249; VI-NEXT: v_min_u16_e32 v0, v0, v1250; VI-NEXT: v_min_u16_e32 v0, v0, v2251; VI-NEXT: s_setpc_b64 s[30:31]252entry:253 %rdx.shuf = shufflevector <4 x i16> %vec4, <4 x i16> poison, <4 x i32> <i32 2, i32 3, i32 poison, i32 poison>254 %rdx.minmax.cmp = icmp ult <4 x i16> %vec4, %rdx.shuf255 %rdx.minmax.select = select <4 x i1> %rdx.minmax.cmp, <4 x i16> %vec4, <4 x i16> %rdx.shuf256 %rdx.shuf1 = shufflevector <4 x i16> %rdx.minmax.select, <4 x i16> poison, <4 x i32> <i32 1, i32 poison, i32 poison, i32 poison>257 %rdx.minmax.cmp2 = icmp ult <4 x i16> %rdx.minmax.select, %rdx.shuf1258 %rdx.minmax.select3 = select <4 x i1> %rdx.minmax.cmp2, <4 x i16> %rdx.minmax.select, <4 x i16> %rdx.shuf1259 %res = extractelement <4 x i16> %rdx.minmax.select3, i32 0260 ret i16 %res261}262 263define i16 @reduction_umin_v8i16(<8 x i16> %vec8) {264; GFX9-LABEL: reduction_umin_v8i16:265; GFX9: ; %bb.0: ; %entry266; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)267; GFX9-NEXT: v_pk_min_u16 v1, v1, v3268; GFX9-NEXT: v_pk_min_u16 v0, v0, v2269; GFX9-NEXT: v_pk_min_u16 v0, v0, v1270; GFX9-NEXT: v_min_u16_sdwa v0, v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1271; GFX9-NEXT: s_setpc_b64 s[30:31]272;273; VI-LABEL: reduction_umin_v8i16:274; VI: ; %bb.0: ; %entry275; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)276; VI-NEXT: v_min_u16_sdwa v4, v1, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1277; VI-NEXT: v_min_u16_sdwa v5, v0, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1278; VI-NEXT: v_min_u16_e32 v1, v1, v3279; VI-NEXT: v_min_u16_e32 v0, v0, v2280; VI-NEXT: v_min_u16_e32 v2, v5, v4281; VI-NEXT: v_min_u16_e32 v0, v0, v1282; VI-NEXT: v_min_u16_e32 v0, v0, v2283; VI-NEXT: s_setpc_b64 s[30:31]284entry:285 %rdx.shuf = shufflevector <8 x i16> %vec8, <8 x i16> poison, <8 x i32> <i32 4, i32 5, i32 6, i32 7, i32 poison, i32 poison, i32 poison, i32 poison>286 %rdx.minmax.cmp = icmp ult <8 x i16> %vec8, %rdx.shuf287 %rdx.minmax.select = select <8 x i1> %rdx.minmax.cmp, <8 x i16> %vec8, <8 x i16> %rdx.shuf288 %rdx.shuf1 = shufflevector <8 x i16> %rdx.minmax.select, <8 x i16> poison, <8 x i32> <i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>289 %rdx.minmax.cmp2 = icmp ult <8 x i16> %rdx.minmax.select, %rdx.shuf1290 %rdx.minmax.select3 = select <8 x i1> %rdx.minmax.cmp2, <8 x i16> %rdx.minmax.select, <8 x i16> %rdx.shuf1291 %rdx.shuf4 = shufflevector <8 x i16> %rdx.minmax.select3, <8 x i16> poison, <8 x i32> <i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>292 %rdx.minmax.cmp5 = icmp ult <8 x i16> %rdx.minmax.select3, %rdx.shuf4293 %rdx.minmax.select6 = select <8 x i1> %rdx.minmax.cmp5, <8 x i16> %rdx.minmax.select3, <8 x i16> %rdx.shuf4294 %res = extractelement <8 x i16> %rdx.minmax.select6, i32 0295 ret i16 %res296}297 298; Tests to make sure without slp the number of instructions are more.299define i16 @reduction_umin_v8i16_woslp(<8 x i16> %vec8) {300; GFX9-LABEL: reduction_umin_v8i16_woslp:301; GFX9: ; %bb.0: ; %entry302; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)303; GFX9-NEXT: v_lshrrev_b32_e32 v4, 16, v1304; GFX9-NEXT: v_min_u16_sdwa v0, v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD305; GFX9-NEXT: v_lshrrev_b32_e32 v5, 16, v2306; GFX9-NEXT: v_min3_u16 v0, v4, v1, v0307; GFX9-NEXT: v_lshrrev_b32_e32 v6, 16, v3308; GFX9-NEXT: v_min3_u16 v0, v5, v2, v0309; GFX9-NEXT: v_min3_u16 v0, v6, v3, v0310; GFX9-NEXT: s_setpc_b64 s[30:31]311;312; VI-LABEL: reduction_umin_v8i16_woslp:313; VI: ; %bb.0: ; %entry314; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)315; VI-NEXT: v_min_u16_sdwa v0, v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD316; VI-NEXT: v_min_u16_e32 v0, v1, v0317; VI-NEXT: v_min_u16_sdwa v0, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD318; VI-NEXT: v_min_u16_e32 v0, v2, v0319; VI-NEXT: v_min_u16_sdwa v0, v2, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD320; VI-NEXT: v_min_u16_e32 v0, v3, v0321; VI-NEXT: v_min_u16_sdwa v0, v3, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD322; VI-NEXT: s_setpc_b64 s[30:31]323entry:324 %elt0 = extractelement <8 x i16> %vec8, i64 0325 %elt1 = extractelement <8 x i16> %vec8, i64 1326 %elt2 = extractelement <8 x i16> %vec8, i64 2327 %elt3 = extractelement <8 x i16> %vec8, i64 3328 %elt4 = extractelement <8 x i16> %vec8, i64 4329 %elt5 = extractelement <8 x i16> %vec8, i64 5330 %elt6 = extractelement <8 x i16> %vec8, i64 6331 %elt7 = extractelement <8 x i16> %vec8, i64 7332 333 %cmp0 = icmp ult i16 %elt1, %elt0334 %min1 = select i1 %cmp0, i16 %elt1, i16 %elt0335 %cmp1 = icmp ult i16 %elt2, %min1336 %min2 = select i1 %cmp1, i16 %elt2, i16 %min1337 %cmp2 = icmp ult i16 %elt3, %min2338 %min3 = select i1 %cmp2, i16 %elt3, i16 %min2339 340 %cmp3 = icmp ult i16 %elt4, %min3341 %min4 = select i1 %cmp3, i16 %elt4, i16 %min3342 %cmp4 = icmp ult i16 %elt5, %min4343 %min5 = select i1 %cmp4, i16 %elt5, i16 %min4344 345 %cmp5 = icmp ult i16 %elt6, %min5346 %min6 = select i1 %cmp5, i16 %elt6, i16 %min5347 %cmp6 = icmp ult i16 %elt7, %min6348 %min7 = select i1 %cmp6, i16 %elt7, i16 %min6349 350 ret i16 %min7351}352 353define i16 @reduction_smin_v16i16(<16 x i16> %vec16) {354; GFX9-LABEL: reduction_smin_v16i16:355; GFX9: ; %bb.0: ; %entry356; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)357; GFX9-NEXT: v_pk_min_i16 v2, v2, v6358; GFX9-NEXT: v_pk_min_i16 v0, v0, v4359; GFX9-NEXT: v_pk_min_i16 v3, v3, v7360; GFX9-NEXT: v_pk_min_i16 v1, v1, v5361; GFX9-NEXT: v_pk_min_i16 v1, v1, v3362; GFX9-NEXT: v_pk_min_i16 v0, v0, v2363; GFX9-NEXT: v_pk_min_i16 v0, v0, v1364; GFX9-NEXT: v_min_i16_sdwa v0, v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1365; GFX9-NEXT: s_setpc_b64 s[30:31]366;367; VI-LABEL: reduction_smin_v16i16:368; VI: ; %bb.0: ; %entry369; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)370; VI-NEXT: v_min_i16_sdwa v8, v2, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1371; VI-NEXT: v_min_i16_sdwa v9, v0, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1372; VI-NEXT: v_min_i16_sdwa v10, v3, v7 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1373; VI-NEXT: v_min_i16_sdwa v11, v1, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1374; VI-NEXT: v_min_i16_e32 v2, v2, v6375; VI-NEXT: v_min_i16_e32 v0, v0, v4376; VI-NEXT: v_min_i16_e32 v3, v3, v7377; VI-NEXT: v_min_i16_e32 v1, v1, v5378; VI-NEXT: v_min_i16_e32 v4, v11, v10379; VI-NEXT: v_min_i16_e32 v5, v9, v8380; VI-NEXT: v_min_i16_e32 v1, v1, v3381; VI-NEXT: v_min_i16_e32 v0, v0, v2382; VI-NEXT: v_min_i16_e32 v2, v5, v4383; VI-NEXT: v_min_i16_e32 v0, v0, v1384; VI-NEXT: v_min_i16_e32 v0, v0, v2385; VI-NEXT: s_setpc_b64 s[30:31]386entry:387 %rdx.shuf = shufflevector <16 x i16> %vec16, <16 x i16> poison, <16 x i32> <i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>388 %rdx.minmax.cmp = icmp slt <16 x i16> %vec16, %rdx.shuf389 %rdx.minmax.select = select <16 x i1> %rdx.minmax.cmp, <16 x i16> %vec16, <16 x i16> %rdx.shuf390 %rdx.shuf1 = shufflevector <16 x i16> %rdx.minmax.select, <16 x i16> poison, <16 x i32> <i32 4, i32 5, i32 6, i32 7, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>391 %rdx.minmax.cmp2 = icmp slt <16 x i16> %rdx.minmax.select, %rdx.shuf1392 %rdx.minmax.select3 = select <16 x i1> %rdx.minmax.cmp2, <16 x i16> %rdx.minmax.select, <16 x i16> %rdx.shuf1393 %rdx.shuf4 = shufflevector <16 x i16> %rdx.minmax.select3, <16 x i16> poison, <16 x i32> <i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>394 %rdx.minmax.cmp5 = icmp slt <16 x i16> %rdx.minmax.select3, %rdx.shuf4395 %rdx.minmax.select6 = select <16 x i1> %rdx.minmax.cmp5, <16 x i16> %rdx.minmax.select3, <16 x i16> %rdx.shuf4396 %rdx.shuf7 = shufflevector <16 x i16> %rdx.minmax.select6, <16 x i16> poison, <16 x i32> <i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>397 %rdx.minmax.cmp8 = icmp slt <16 x i16> %rdx.minmax.select6, %rdx.shuf7398 %rdx.minmax.select9 = select <16 x i1> %rdx.minmax.cmp8, <16 x i16> %rdx.minmax.select6, <16 x i16> %rdx.shuf7399 %res = extractelement <16 x i16> %rdx.minmax.select9, i32 0400 ret i16 %res401}402 403; Tests to make sure without slp the number of instructions are more.404define i16 @reduction_smin_v16i16_woslp(<16 x i16> %vec16) {405; GFX9-LABEL: reduction_smin_v16i16_woslp:406; GFX9: ; %bb.0: ; %entry407; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)408; GFX9-NEXT: v_lshrrev_b32_e32 v8, 16, v1409; GFX9-NEXT: v_min_i16_sdwa v0, v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD410; GFX9-NEXT: v_lshrrev_b32_e32 v9, 16, v2411; GFX9-NEXT: v_min3_i16 v0, v8, v1, v0412; GFX9-NEXT: v_lshrrev_b32_e32 v10, 16, v3413; GFX9-NEXT: v_min3_i16 v0, v9, v2, v0414; GFX9-NEXT: v_lshrrev_b32_e32 v11, 16, v4415; GFX9-NEXT: v_min3_i16 v0, v10, v3, v0416; GFX9-NEXT: v_lshrrev_b32_e32 v12, 16, v5417; GFX9-NEXT: v_min3_i16 v0, v11, v4, v0418; GFX9-NEXT: v_lshrrev_b32_e32 v13, 16, v6419; GFX9-NEXT: v_min3_i16 v0, v12, v5, v0420; GFX9-NEXT: v_lshrrev_b32_e32 v14, 16, v7421; GFX9-NEXT: v_min3_i16 v0, v13, v6, v0422; GFX9-NEXT: v_min3_i16 v0, v14, v7, v0423; GFX9-NEXT: s_setpc_b64 s[30:31]424;425; VI-LABEL: reduction_smin_v16i16_woslp:426; VI: ; %bb.0: ; %entry427; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)428; VI-NEXT: v_min_i16_sdwa v0, v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD429; VI-NEXT: v_min_i16_e32 v0, v1, v0430; VI-NEXT: v_min_i16_sdwa v0, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD431; VI-NEXT: v_min_i16_e32 v0, v2, v0432; VI-NEXT: v_min_i16_sdwa v0, v2, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD433; VI-NEXT: v_min_i16_e32 v0, v3, v0434; VI-NEXT: v_min_i16_sdwa v0, v3, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD435; VI-NEXT: v_min_i16_e32 v0, v4, v0436; VI-NEXT: v_min_i16_sdwa v0, v4, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD437; VI-NEXT: v_min_i16_e32 v0, v5, v0438; VI-NEXT: v_min_i16_sdwa v0, v5, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD439; VI-NEXT: v_min_i16_e32 v0, v6, v0440; VI-NEXT: v_min_i16_sdwa v0, v6, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD441; VI-NEXT: v_min_i16_e32 v0, v7, v0442; VI-NEXT: v_min_i16_sdwa v0, v7, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD443; VI-NEXT: s_setpc_b64 s[30:31]444entry:445 %elt0 = extractelement <16 x i16> %vec16, i64 0446 %elt1 = extractelement <16 x i16> %vec16, i64 1447 %elt2 = extractelement <16 x i16> %vec16, i64 2448 %elt3 = extractelement <16 x i16> %vec16, i64 3449 %elt4 = extractelement <16 x i16> %vec16, i64 4450 %elt5 = extractelement <16 x i16> %vec16, i64 5451 %elt6 = extractelement <16 x i16> %vec16, i64 6452 %elt7 = extractelement <16 x i16> %vec16, i64 7453 454 %elt8 = extractelement <16 x i16> %vec16, i64 8455 %elt9 = extractelement <16 x i16> %vec16, i64 9456 %elt10 = extractelement <16 x i16> %vec16, i64 10457 %elt11 = extractelement <16 x i16> %vec16, i64 11458 %elt12 = extractelement <16 x i16> %vec16, i64 12459 %elt13 = extractelement <16 x i16> %vec16, i64 13460 %elt14 = extractelement <16 x i16> %vec16, i64 14461 %elt15 = extractelement <16 x i16> %vec16, i64 15462 463 %cmp0 = icmp slt i16 %elt1, %elt0464 %min1 = select i1 %cmp0, i16 %elt1, i16 %elt0465 %cmp1 = icmp slt i16 %elt2, %min1466 %min2 = select i1 %cmp1, i16 %elt2, i16 %min1467 %cmp2 = icmp slt i16 %elt3, %min2468 %min3 = select i1 %cmp2, i16 %elt3, i16 %min2469 470 %cmp3 = icmp slt i16 %elt4, %min3471 %min4 = select i1 %cmp3, i16 %elt4, i16 %min3472 %cmp4 = icmp slt i16 %elt5, %min4473 %min5 = select i1 %cmp4, i16 %elt5, i16 %min4474 475 %cmp5 = icmp slt i16 %elt6, %min5476 %min6 = select i1 %cmp5, i16 %elt6, i16 %min5477 %cmp6 = icmp slt i16 %elt7, %min6478 %min7 = select i1 %cmp6, i16 %elt7, i16 %min6479 480 %cmp7 = icmp slt i16 %elt8, %min7481 %min8 = select i1 %cmp7, i16 %elt8, i16 %min7482 %cmp8 = icmp slt i16 %elt9, %min8483 %min9 = select i1 %cmp8, i16 %elt9, i16 %min8484 485 %cmp9 = icmp slt i16 %elt10, %min9486 %min10 = select i1 %cmp9, i16 %elt10, i16 %min9487 %cmp10 = icmp slt i16 %elt11, %min10488 %min11 = select i1 %cmp10, i16 %elt11, i16 %min10489 490 %cmp11 = icmp slt i16 %elt12, %min11491 %min12 = select i1 %cmp11, i16 %elt12, i16 %min11492 %cmp12 = icmp slt i16 %elt13, %min12493 %min13 = select i1 %cmp12, i16 %elt13, i16 %min12494 495 %cmp13 = icmp slt i16 %elt14, %min13496 %min14 = select i1 %cmp13, i16 %elt14, i16 %min13497 %cmp14 = icmp slt i16 %elt15, %min14498 %min15 = select i1 %cmp14, i16 %elt15, i16 %min14499 500 501 ret i16 %min15502}503 504define i16 @reduction_umax_v4i16(<4 x i16> %vec4) {505; GFX9-LABEL: reduction_umax_v4i16:506; GFX9: ; %bb.0: ; %entry507; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)508; GFX9-NEXT: v_pk_max_u16 v0, v0, v1509; GFX9-NEXT: v_max_u16_sdwa v0, v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1510; GFX9-NEXT: s_setpc_b64 s[30:31]511;512; VI-LABEL: reduction_umax_v4i16:513; VI: ; %bb.0: ; %entry514; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)515; VI-NEXT: v_max_u16_sdwa v2, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1516; VI-NEXT: v_max_u16_e32 v0, v0, v1517; VI-NEXT: v_max_u16_e32 v0, v0, v2518; VI-NEXT: s_setpc_b64 s[30:31]519entry:520 %rdx.shuf = shufflevector <4 x i16> %vec4, <4 x i16> poison, <4 x i32> <i32 2, i32 3, i32 poison, i32 poison>521 %rdx.minmax.cmp = icmp ugt <4 x i16> %vec4, %rdx.shuf522 %rdx.minmax.select = select <4 x i1> %rdx.minmax.cmp, <4 x i16> %vec4, <4 x i16> %rdx.shuf523 %rdx.shuf1 = shufflevector <4 x i16> %rdx.minmax.select, <4 x i16> poison, <4 x i32> <i32 1, i32 poison, i32 poison, i32 poison>524 %rdx.minmax.cmp2 = icmp ugt <4 x i16> %rdx.minmax.select, %rdx.shuf1525 %rdx.minmax.select3 = select <4 x i1> %rdx.minmax.cmp2, <4 x i16> %rdx.minmax.select, <4 x i16> %rdx.shuf1526 %res = extractelement <4 x i16> %rdx.minmax.select3, i32 0527 ret i16 %res528}529 530define i16 @reduction_smax_v4i16(<4 x i16> %vec4) #0 {531; GFX9-LABEL: reduction_smax_v4i16:532; GFX9: ; %bb.0: ; %entry533; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)534; GFX9-NEXT: v_pk_max_i16 v0, v0, v1535; GFX9-NEXT: v_max_i16_sdwa v0, v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1536; GFX9-NEXT: s_setpc_b64 s[30:31]537;538; VI-LABEL: reduction_smax_v4i16:539; VI: ; %bb.0: ; %entry540; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)541; VI-NEXT: v_max_i16_sdwa v2, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1542; VI-NEXT: v_max_i16_e32 v0, v0, v1543; VI-NEXT: v_max_i16_e32 v0, v0, v2544; VI-NEXT: s_setpc_b64 s[30:31]545entry:546 %rdx.shuf = shufflevector <4 x i16> %vec4, <4 x i16> poison, <4 x i32> <i32 2, i32 3, i32 poison, i32 poison>547 %rdx.minmax.cmp = icmp sgt <4 x i16> %vec4, %rdx.shuf548 %rdx.minmax.select = select <4 x i1> %rdx.minmax.cmp, <4 x i16> %vec4, <4 x i16> %rdx.shuf549 %rdx.shuf1 = shufflevector <4 x i16> %rdx.minmax.select, <4 x i16> poison, <4 x i32> <i32 1, i32 poison, i32 poison, i32 poison>550 %rdx.minmax.cmp2 = icmp sgt <4 x i16> %rdx.minmax.select, %rdx.shuf1551 %rdx.minmax.select3 = select <4 x i1> %rdx.minmax.cmp2, <4 x i16> %rdx.minmax.select, <4 x i16> %rdx.shuf1552 %res = extractelement <4 x i16> %rdx.minmax.select3, i32 0553 ret i16 %res554}555 556define half @reduction_maxnum_v4f16(<4 x half> %vec4) {557; GFX9-LABEL: reduction_maxnum_v4f16:558; GFX9: ; %bb.0: ; %entry559; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)560; GFX9-NEXT: v_pk_max_f16 v1, v1, v1561; GFX9-NEXT: v_pk_max_f16 v0, v0, v0562; GFX9-NEXT: v_pk_max_f16 v0, v0, v1563; GFX9-NEXT: v_max_f16_sdwa v0, v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1564; GFX9-NEXT: s_setpc_b64 s[30:31]565;566; VI-LABEL: reduction_maxnum_v4f16:567; VI: ; %bb.0: ; %entry568; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)569; VI-NEXT: v_max_f16_sdwa v2, v1, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1570; VI-NEXT: v_max_f16_sdwa v3, v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1571; VI-NEXT: v_max_f16_e32 v1, v1, v1572; VI-NEXT: v_max_f16_e32 v0, v0, v0573; VI-NEXT: v_max_f16_e32 v2, v3, v2574; VI-NEXT: v_max_f16_e32 v0, v0, v1575; VI-NEXT: v_max_f16_e32 v0, v0, v2576; VI-NEXT: s_setpc_b64 s[30:31]577entry:578 %rdx.shuf = shufflevector <4 x half> %vec4, <4 x half> poison, <4 x i32> <i32 2, i32 3, i32 poison, i32 poison>579 %rdx.minmax = call <4 x half> @llvm.maxnum.v4f16(<4 x half> %vec4, <4 x half> %rdx.shuf)580 %rdx.shuf1 = shufflevector <4 x half> %rdx.minmax, <4 x half> poison, <4 x i32> <i32 1, i32 poison, i32 poison, i32 poison>581 %rdx.minmax3 = call <4 x half> @llvm.maxnum.v4f16(<4 x half> %rdx.minmax, <4 x half> %rdx.shuf1)582 %res = extractelement <4 x half> %rdx.minmax3, i32 0583 ret half %res584}585 586define half @reduction_minnum_v4f16(<4 x half> %vec4) {587; GFX9-LABEL: reduction_minnum_v4f16:588; GFX9: ; %bb.0: ; %entry589; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)590; GFX9-NEXT: v_pk_max_f16 v1, v1, v1591; GFX9-NEXT: v_pk_max_f16 v0, v0, v0592; GFX9-NEXT: v_pk_min_f16 v0, v0, v1593; GFX9-NEXT: v_min_f16_sdwa v0, v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1594; GFX9-NEXT: s_setpc_b64 s[30:31]595;596; VI-LABEL: reduction_minnum_v4f16:597; VI: ; %bb.0: ; %entry598; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)599; VI-NEXT: v_max_f16_sdwa v2, v1, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1600; VI-NEXT: v_max_f16_sdwa v3, v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1601; VI-NEXT: v_max_f16_e32 v1, v1, v1602; VI-NEXT: v_max_f16_e32 v0, v0, v0603; VI-NEXT: v_min_f16_e32 v2, v3, v2604; VI-NEXT: v_min_f16_e32 v0, v0, v1605; VI-NEXT: v_min_f16_e32 v0, v0, v2606; VI-NEXT: s_setpc_b64 s[30:31]607entry:608 %rdx.shuf = shufflevector <4 x half> %vec4, <4 x half> poison, <4 x i32> <i32 2, i32 3, i32 poison, i32 poison>609 %rdx.minmax = call <4 x half> @llvm.minnum.v4f16(<4 x half> %vec4, <4 x half> %rdx.shuf)610 %rdx.shuf1 = shufflevector <4 x half> %rdx.minmax, <4 x half> poison, <4 x i32> <i32 1, i32 poison, i32 poison, i32 poison>611 %rdx.minmax3 = call <4 x half> @llvm.minnum.v4f16(<4 x half> %rdx.minmax, <4 x half> %rdx.shuf1)612 %res = extractelement <4 x half> %rdx.minmax3, i32 0613 ret half %res614}615 616; FIXME: Need to preserve fast math flags when fmaxnum matched617; directly from the IR to avoid unnecessary quieting.618define half @reduction_fast_max_pattern_v4f16(<4 x half> %vec4) {619; GFX9-LABEL: reduction_fast_max_pattern_v4f16:620; GFX9: ; %bb.0: ; %entry621; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)622; GFX9-NEXT: v_pk_max_f16 v1, v1, v1623; GFX9-NEXT: v_pk_max_f16 v0, v0, v0624; GFX9-NEXT: v_pk_max_f16 v0, v0, v1625; GFX9-NEXT: v_max_f16_sdwa v0, v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1626; GFX9-NEXT: s_setpc_b64 s[30:31]627;628; VI-LABEL: reduction_fast_max_pattern_v4f16:629; VI: ; %bb.0: ; %entry630; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)631; VI-NEXT: v_max_f16_sdwa v2, v1, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1632; VI-NEXT: v_max_f16_sdwa v3, v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1633; VI-NEXT: v_max_f16_e32 v1, v1, v1634; VI-NEXT: v_max_f16_e32 v0, v0, v0635; VI-NEXT: v_max_f16_e32 v2, v3, v2636; VI-NEXT: v_max_f16_e32 v0, v0, v1637; VI-NEXT: v_max_f16_e32 v0, v0, v2638; VI-NEXT: s_setpc_b64 s[30:31]639entry:640 %rdx.shuf = shufflevector <4 x half> %vec4, <4 x half> poison, <4 x i32> <i32 2, i32 3, i32 poison, i32 poison>641 %rdx.minmax.cmp = fcmp nnan nsz ogt <4 x half> %vec4, %rdx.shuf642 %rdx.minmax.select = select <4 x i1> %rdx.minmax.cmp, <4 x half> %vec4, <4 x half> %rdx.shuf643 %rdx.shuf1 = shufflevector <4 x half> %rdx.minmax.select, <4 x half> poison, <4 x i32> <i32 1, i32 poison, i32 poison, i32 poison>644 %rdx.minmax.cmp2 = fcmp nnan nsz ogt <4 x half> %rdx.minmax.select, %rdx.shuf1645 %rdx.minmax.select3 = select <4 x i1> %rdx.minmax.cmp2, <4 x half> %rdx.minmax.select, <4 x half> %rdx.shuf1646 %res = extractelement <4 x half> %rdx.minmax.select3, i32 0647 ret half %res648}649 650; FIXME: Need to preserve fast math flags when fmaxnum matched651; directly from the IR to avoid unnecessary quieting.652define half @reduction_fast_min_pattern_v4f16(<4 x half> %vec4) {653; GFX9-LABEL: reduction_fast_min_pattern_v4f16:654; GFX9: ; %bb.0: ; %entry655; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)656; GFX9-NEXT: v_pk_max_f16 v1, v1, v1657; GFX9-NEXT: v_pk_max_f16 v0, v0, v0658; GFX9-NEXT: v_pk_min_f16 v0, v0, v1659; GFX9-NEXT: v_min_f16_sdwa v0, v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1660; GFX9-NEXT: s_setpc_b64 s[30:31]661;662; VI-LABEL: reduction_fast_min_pattern_v4f16:663; VI: ; %bb.0: ; %entry664; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)665; VI-NEXT: v_max_f16_sdwa v2, v1, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1666; VI-NEXT: v_max_f16_sdwa v3, v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1667; VI-NEXT: v_max_f16_e32 v1, v1, v1668; VI-NEXT: v_max_f16_e32 v0, v0, v0669; VI-NEXT: v_min_f16_e32 v2, v3, v2670; VI-NEXT: v_min_f16_e32 v0, v0, v1671; VI-NEXT: v_min_f16_e32 v0, v0, v2672; VI-NEXT: s_setpc_b64 s[30:31]673entry:674 %rdx.shuf = shufflevector <4 x half> %vec4, <4 x half> poison, <4 x i32> <i32 2, i32 3, i32 poison, i32 poison>675 %rdx.minmax.cmp = fcmp nnan nsz olt <4 x half> %vec4, %rdx.shuf676 %rdx.minmax.select = select <4 x i1> %rdx.minmax.cmp, <4 x half> %vec4, <4 x half> %rdx.shuf677 %rdx.shuf1 = shufflevector <4 x half> %rdx.minmax.select, <4 x half> poison, <4 x i32> <i32 1, i32 poison, i32 poison, i32 poison>678 %rdx.minmax.cmp2 = fcmp nnan nsz olt <4 x half> %rdx.minmax.select, %rdx.shuf1679 %rdx.minmax.select3 = select <4 x i1> %rdx.minmax.cmp2, <4 x half> %rdx.minmax.select, <4 x half> %rdx.shuf1680 %res = extractelement <4 x half> %rdx.minmax.select3, i32 0681 ret half %res682}683 684declare <4 x half> @llvm.minnum.v4f16(<4 x half>, <4 x half>)685declare <4 x half> @llvm.maxnum.v4f16(<4 x half>, <4 x half>)686