414 lines · plain
1; NOTE: Assertions have been autogenerated by utils/update_test_checks.py2; RUN: opt -passes='require<profile-summary>,function(codegenprepare)' -mtriple=x86_64-- -mattr=+avx -S < %s | FileCheck %s --check-prefixes=AVX13; RUN: opt -passes='require<profile-summary>,function(codegenprepare)' -mtriple=x86_64-- -mattr=+avx2 -S < %s | FileCheck %s --check-prefixes=AVX24; RUN: opt -passes='require<profile-summary>,function(codegenprepare)' -mtriple=x86_64-- -mattr=+avx512bw -S < %s | FileCheck %s --check-prefixes=AVX512BW5; RUN: opt -passes='require<profile-summary>,function(codegenprepare)' -mtriple=x86_64-- -mattr=+avx,+xop -S < %s | FileCheck %s --check-prefixes=XOP6; RUN: opt -passes='require<profile-summary>,function(codegenprepare)' -mtriple=x86_64-- -mattr=+avx2,+xop -S < %s | FileCheck %s --check-prefixes=XOP7; RUN: opt -passes='require<profile-summary>,function(codegenprepare)' -mtriple=x86_64-- -mattr=+avx -S -enable-debugify < %s 2>&1 | FileCheck %s -check-prefix=DEBUG8 9define <4 x i32> @vector_variable_shift_right_v4i32(<4 x i1> %cond, <4 x i32> %x, <4 x i32> %y, <4 x i32> %z) {10; AVX1-LABEL: @vector_variable_shift_right_v4i32(11; AVX1-NEXT: [[SPLAT1:%.*]] = shufflevector <4 x i32> [[X:%.*]], <4 x i32> undef, <4 x i32> zeroinitializer12; AVX1-NEXT: [[SPLAT2:%.*]] = shufflevector <4 x i32> [[Y:%.*]], <4 x i32> undef, <4 x i32> zeroinitializer13; AVX1-NEXT: [[SEL:%.*]] = select <4 x i1> [[COND:%.*]], <4 x i32> [[SPLAT1]], <4 x i32> [[SPLAT2]]14; AVX1-NEXT: [[TMP1:%.*]] = lshr <4 x i32> [[Z:%.*]], [[SPLAT1]]15; AVX1-NEXT: [[TMP2:%.*]] = lshr <4 x i32> [[Z]], [[SPLAT2]]16; AVX1-NEXT: [[TMP3:%.*]] = select <4 x i1> [[COND]], <4 x i32> [[TMP1]], <4 x i32> [[TMP2]]17; AVX1-NEXT: ret <4 x i32> [[TMP3]]18;19; AVX2-LABEL: @vector_variable_shift_right_v4i32(20; AVX2-NEXT: [[SPLAT1:%.*]] = shufflevector <4 x i32> [[X:%.*]], <4 x i32> undef, <4 x i32> zeroinitializer21; AVX2-NEXT: [[SPLAT2:%.*]] = shufflevector <4 x i32> [[Y:%.*]], <4 x i32> undef, <4 x i32> zeroinitializer22; AVX2-NEXT: [[SEL:%.*]] = select <4 x i1> [[COND:%.*]], <4 x i32> [[SPLAT1]], <4 x i32> [[SPLAT2]]23; AVX2-NEXT: [[SH:%.*]] = lshr <4 x i32> [[Z:%.*]], [[SEL]]24; AVX2-NEXT: ret <4 x i32> [[SH]]25;26; AVX512BW-LABEL: @vector_variable_shift_right_v4i32(27; AVX512BW-NEXT: [[SPLAT1:%.*]] = shufflevector <4 x i32> [[X:%.*]], <4 x i32> undef, <4 x i32> zeroinitializer28; AVX512BW-NEXT: [[SPLAT2:%.*]] = shufflevector <4 x i32> [[Y:%.*]], <4 x i32> undef, <4 x i32> zeroinitializer29; AVX512BW-NEXT: [[SEL:%.*]] = select <4 x i1> [[COND:%.*]], <4 x i32> [[SPLAT1]], <4 x i32> [[SPLAT2]]30; AVX512BW-NEXT: [[SH:%.*]] = lshr <4 x i32> [[Z:%.*]], [[SEL]]31; AVX512BW-NEXT: ret <4 x i32> [[SH]]32;33; XOP-LABEL: @vector_variable_shift_right_v4i32(34; XOP-NEXT: [[SPLAT1:%.*]] = shufflevector <4 x i32> [[X:%.*]], <4 x i32> undef, <4 x i32> zeroinitializer35; XOP-NEXT: [[SPLAT2:%.*]] = shufflevector <4 x i32> [[Y:%.*]], <4 x i32> undef, <4 x i32> zeroinitializer36; XOP-NEXT: [[SEL:%.*]] = select <4 x i1> [[COND:%.*]], <4 x i32> [[SPLAT1]], <4 x i32> [[SPLAT2]]37; XOP-NEXT: [[SH:%.*]] = lshr <4 x i32> [[Z:%.*]], [[SEL]]38; XOP-NEXT: ret <4 x i32> [[SH]]39;40 %splat1 = shufflevector <4 x i32> %x, <4 x i32> undef, <4 x i32> zeroinitializer41 %splat2 = shufflevector <4 x i32> %y, <4 x i32> undef, <4 x i32> zeroinitializer42 %sel = select <4 x i1> %cond, <4 x i32> %splat1, <4 x i32> %splat243 %sh = lshr <4 x i32> %z, %sel44 ret <4 x i32> %sh45}46 47define <16 x i16> @vector_variable_shift_right_v16i16(<16 x i1> %cond, <16 x i16> %x, <16 x i16> %y, <16 x i16> %z) {48; AVX1-LABEL: @vector_variable_shift_right_v16i16(49; AVX1-NEXT: [[SPLAT1:%.*]] = shufflevector <16 x i16> [[X:%.*]], <16 x i16> undef, <16 x i32> zeroinitializer50; AVX1-NEXT: [[SPLAT2:%.*]] = shufflevector <16 x i16> [[Y:%.*]], <16 x i16> undef, <16 x i32> zeroinitializer51; AVX1-NEXT: [[SEL:%.*]] = select <16 x i1> [[COND:%.*]], <16 x i16> [[SPLAT1]], <16 x i16> [[SPLAT2]]52; AVX1-NEXT: [[TMP1:%.*]] = lshr <16 x i16> [[Z:%.*]], [[SPLAT1]]53; AVX1-NEXT: [[TMP2:%.*]] = lshr <16 x i16> [[Z]], [[SPLAT2]]54; AVX1-NEXT: [[TMP3:%.*]] = select <16 x i1> [[COND]], <16 x i16> [[TMP1]], <16 x i16> [[TMP2]]55; AVX1-NEXT: ret <16 x i16> [[TMP3]]56;57; AVX2-LABEL: @vector_variable_shift_right_v16i16(58; AVX2-NEXT: [[SPLAT1:%.*]] = shufflevector <16 x i16> [[X:%.*]], <16 x i16> undef, <16 x i32> zeroinitializer59; AVX2-NEXT: [[SPLAT2:%.*]] = shufflevector <16 x i16> [[Y:%.*]], <16 x i16> undef, <16 x i32> zeroinitializer60; AVX2-NEXT: [[SEL:%.*]] = select <16 x i1> [[COND:%.*]], <16 x i16> [[SPLAT1]], <16 x i16> [[SPLAT2]]61; AVX2-NEXT: [[TMP1:%.*]] = lshr <16 x i16> [[Z:%.*]], [[SPLAT1]]62; AVX2-NEXT: [[TMP2:%.*]] = lshr <16 x i16> [[Z]], [[SPLAT2]]63; AVX2-NEXT: [[TMP3:%.*]] = select <16 x i1> [[COND]], <16 x i16> [[TMP1]], <16 x i16> [[TMP2]]64; AVX2-NEXT: ret <16 x i16> [[TMP3]]65;66; AVX512BW-LABEL: @vector_variable_shift_right_v16i16(67; AVX512BW-NEXT: [[SPLAT1:%.*]] = shufflevector <16 x i16> [[X:%.*]], <16 x i16> undef, <16 x i32> zeroinitializer68; AVX512BW-NEXT: [[SPLAT2:%.*]] = shufflevector <16 x i16> [[Y:%.*]], <16 x i16> undef, <16 x i32> zeroinitializer69; AVX512BW-NEXT: [[SEL:%.*]] = select <16 x i1> [[COND:%.*]], <16 x i16> [[SPLAT1]], <16 x i16> [[SPLAT2]]70; AVX512BW-NEXT: [[SH:%.*]] = lshr <16 x i16> [[Z:%.*]], [[SEL]]71; AVX512BW-NEXT: ret <16 x i16> [[SH]]72;73; XOP-LABEL: @vector_variable_shift_right_v16i16(74; XOP-NEXT: [[SPLAT1:%.*]] = shufflevector <16 x i16> [[X:%.*]], <16 x i16> undef, <16 x i32> zeroinitializer75; XOP-NEXT: [[SPLAT2:%.*]] = shufflevector <16 x i16> [[Y:%.*]], <16 x i16> undef, <16 x i32> zeroinitializer76; XOP-NEXT: [[SEL:%.*]] = select <16 x i1> [[COND:%.*]], <16 x i16> [[SPLAT1]], <16 x i16> [[SPLAT2]]77; XOP-NEXT: [[SH:%.*]] = lshr <16 x i16> [[Z:%.*]], [[SEL]]78; XOP-NEXT: ret <16 x i16> [[SH]]79;80 %splat1 = shufflevector <16 x i16> %x, <16 x i16> undef, <16 x i32> zeroinitializer81 %splat2 = shufflevector <16 x i16> %y, <16 x i16> undef, <16 x i32> zeroinitializer82 %sel = select <16 x i1> %cond, <16 x i16> %splat1, <16 x i16> %splat283 %sh = lshr <16 x i16> %z, %sel84 ret <16 x i16> %sh85}86 87define <32 x i8> @vector_variable_shift_right_v32i8(<32 x i1> %cond, <32 x i8> %x, <32 x i8> %y, <32 x i8> %z) {88; AVX1-LABEL: @vector_variable_shift_right_v32i8(89; AVX1-NEXT: [[SPLAT1:%.*]] = shufflevector <32 x i8> [[X:%.*]], <32 x i8> undef, <32 x i32> zeroinitializer90; AVX1-NEXT: [[SPLAT2:%.*]] = shufflevector <32 x i8> [[Y:%.*]], <32 x i8> undef, <32 x i32> zeroinitializer91; AVX1-NEXT: [[SEL:%.*]] = select <32 x i1> [[COND:%.*]], <32 x i8> [[SPLAT1]], <32 x i8> [[SPLAT2]]92; AVX1-NEXT: [[TMP1:%.*]] = lshr <32 x i8> [[Z:%.*]], [[SPLAT1]]93; AVX1-NEXT: [[TMP2:%.*]] = lshr <32 x i8> [[Z]], [[SPLAT2]]94; AVX1-NEXT: [[TMP3:%.*]] = select <32 x i1> [[COND]], <32 x i8> [[TMP1]], <32 x i8> [[TMP2]]95; AVX1-NEXT: ret <32 x i8> [[TMP3]]96;97; AVX2-LABEL: @vector_variable_shift_right_v32i8(98; AVX2-NEXT: [[SPLAT1:%.*]] = shufflevector <32 x i8> [[X:%.*]], <32 x i8> undef, <32 x i32> zeroinitializer99; AVX2-NEXT: [[SPLAT2:%.*]] = shufflevector <32 x i8> [[Y:%.*]], <32 x i8> undef, <32 x i32> zeroinitializer100; AVX2-NEXT: [[SEL:%.*]] = select <32 x i1> [[COND:%.*]], <32 x i8> [[SPLAT1]], <32 x i8> [[SPLAT2]]101; AVX2-NEXT: [[TMP1:%.*]] = lshr <32 x i8> [[Z:%.*]], [[SPLAT1]]102; AVX2-NEXT: [[TMP2:%.*]] = lshr <32 x i8> [[Z]], [[SPLAT2]]103; AVX2-NEXT: [[TMP3:%.*]] = select <32 x i1> [[COND]], <32 x i8> [[TMP1]], <32 x i8> [[TMP2]]104; AVX2-NEXT: ret <32 x i8> [[TMP3]]105;106; AVX512BW-LABEL: @vector_variable_shift_right_v32i8(107; AVX512BW-NEXT: [[SPLAT1:%.*]] = shufflevector <32 x i8> [[X:%.*]], <32 x i8> undef, <32 x i32> zeroinitializer108; AVX512BW-NEXT: [[SPLAT2:%.*]] = shufflevector <32 x i8> [[Y:%.*]], <32 x i8> undef, <32 x i32> zeroinitializer109; AVX512BW-NEXT: [[SEL:%.*]] = select <32 x i1> [[COND:%.*]], <32 x i8> [[SPLAT1]], <32 x i8> [[SPLAT2]]110; AVX512BW-NEXT: [[TMP1:%.*]] = lshr <32 x i8> [[Z:%.*]], [[SPLAT1]]111; AVX512BW-NEXT: [[TMP2:%.*]] = lshr <32 x i8> [[Z]], [[SPLAT2]]112; AVX512BW-NEXT: [[TMP3:%.*]] = select <32 x i1> [[COND]], <32 x i8> [[TMP1]], <32 x i8> [[TMP2]]113; AVX512BW-NEXT: ret <32 x i8> [[TMP3]]114;115; XOP-LABEL: @vector_variable_shift_right_v32i8(116; XOP-NEXT: [[SPLAT1:%.*]] = shufflevector <32 x i8> [[X:%.*]], <32 x i8> undef, <32 x i32> zeroinitializer117; XOP-NEXT: [[SPLAT2:%.*]] = shufflevector <32 x i8> [[Y:%.*]], <32 x i8> undef, <32 x i32> zeroinitializer118; XOP-NEXT: [[SEL:%.*]] = select <32 x i1> [[COND:%.*]], <32 x i8> [[SPLAT1]], <32 x i8> [[SPLAT2]]119; XOP-NEXT: [[SH:%.*]] = lshr <32 x i8> [[Z:%.*]], [[SEL]]120; XOP-NEXT: ret <32 x i8> [[SH]]121;122 %splat1 = shufflevector <32 x i8> %x, <32 x i8> undef, <32 x i32> zeroinitializer123 %splat2 = shufflevector <32 x i8> %y, <32 x i8> undef, <32 x i32> zeroinitializer124 %sel = select <32 x i1> %cond, <32 x i8> %splat1, <32 x i8> %splat2125 %sh = lshr <32 x i8> %z, %sel126 ret <32 x i8> %sh127}128 129; PR37428 - https://bugs.llvm.org/show_bug.cgi?id=37428130 131define void @vector_variable_shift_left_loop(ptr nocapture %arr, ptr nocapture readonly %control, i32 %count, i32 %amt0, i32 %amt1, i32 %x) {132; AVX1-LABEL: @vector_variable_shift_left_loop(133; AVX1-NEXT: entry:134; AVX1-NEXT: [[CMP16:%.*]] = icmp sgt i32 [[COUNT:%.*]], 0135; AVX1-NEXT: [[WIDE_TRIP_COUNT:%.*]] = zext i32 [[COUNT]] to i64136; AVX1-NEXT: br i1 [[CMP16]], label [[VECTOR_PH:%.*]], label [[EXIT:%.*]]137; AVX1: vector.ph:138; AVX1-NEXT: [[N_VEC:%.*]] = and i64 [[WIDE_TRIP_COUNT]], 4294967292139; AVX1-NEXT: [[SPLATINSERT18:%.*]] = insertelement <4 x i32> undef, i32 [[AMT0:%.*]], i32 0140; AVX1-NEXT: [[SPLAT1:%.*]] = shufflevector <4 x i32> [[SPLATINSERT18]], <4 x i32> undef, <4 x i32> zeroinitializer141; AVX1-NEXT: [[SPLATINSERT20:%.*]] = insertelement <4 x i32> undef, i32 [[AMT1:%.*]], i32 0142; AVX1-NEXT: [[SPLAT2:%.*]] = shufflevector <4 x i32> [[SPLATINSERT20]], <4 x i32> undef, <4 x i32> zeroinitializer143; AVX1-NEXT: [[SPLATINSERT22:%.*]] = insertelement <4 x i32> undef, i32 [[X:%.*]], i32 0144; AVX1-NEXT: [[SPLAT3:%.*]] = shufflevector <4 x i32> [[SPLATINSERT22]], <4 x i32> undef, <4 x i32> zeroinitializer145; AVX1-NEXT: br label [[VECTOR_BODY:%.*]]146; AVX1: vector.body:147; AVX1-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]148; AVX1-NEXT: [[TMP0:%.*]] = getelementptr inbounds i8, ptr [[CONTROL:%.*]], i64 [[INDEX]]149; AVX1-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i8>, ptr [[TMP0]], align 1150; AVX1-NEXT: [[TMP2:%.*]] = icmp eq <4 x i8> [[WIDE_LOAD]], zeroinitializer151; AVX1-NEXT: [[TMP3:%.*]] = select <4 x i1> [[TMP2]], <4 x i32> [[SPLAT1]], <4 x i32> [[SPLAT2]]152; AVX1-NEXT: [[TMP4:%.*]] = shufflevector <4 x i32> [[SPLATINSERT18]], <4 x i32> undef, <4 x i32> zeroinitializer153; AVX1-NEXT: [[TMP5:%.*]] = shl <4 x i32> [[SPLAT3]], [[TMP4]]154; AVX1-NEXT: [[TMP6:%.*]] = shufflevector <4 x i32> [[SPLATINSERT20]], <4 x i32> undef, <4 x i32> zeroinitializer155; AVX1-NEXT: [[TMP7:%.*]] = shl <4 x i32> [[SPLAT3]], [[TMP6]]156; AVX1-NEXT: [[TMP8:%.*]] = select <4 x i1> [[TMP2]], <4 x i32> [[TMP5]], <4 x i32> [[TMP7]]157; AVX1-NEXT: [[TMP9:%.*]] = getelementptr inbounds i32, ptr [[ARR:%.*]], i64 [[INDEX]]158; AVX1-NEXT: store <4 x i32> [[TMP8]], ptr [[TMP9]], align 4159; AVX1-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], 4160; AVX1-NEXT: [[TMP11:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]161; AVX1-NEXT: br i1 [[TMP11]], label [[EXIT]], label [[VECTOR_BODY]]162; AVX1: exit:163; AVX1-NEXT: ret void164;165; AVX2-LABEL: @vector_variable_shift_left_loop(166; AVX2-NEXT: entry:167; AVX2-NEXT: [[CMP16:%.*]] = icmp sgt i32 [[COUNT:%.*]], 0168; AVX2-NEXT: [[WIDE_TRIP_COUNT:%.*]] = zext i32 [[COUNT]] to i64169; AVX2-NEXT: br i1 [[CMP16]], label [[VECTOR_PH:%.*]], label [[EXIT:%.*]]170; AVX2: vector.ph:171; AVX2-NEXT: [[N_VEC:%.*]] = and i64 [[WIDE_TRIP_COUNT]], 4294967292172; AVX2-NEXT: [[SPLATINSERT18:%.*]] = insertelement <4 x i32> undef, i32 [[AMT0:%.*]], i32 0173; AVX2-NEXT: [[SPLAT1:%.*]] = shufflevector <4 x i32> [[SPLATINSERT18]], <4 x i32> undef, <4 x i32> zeroinitializer174; AVX2-NEXT: [[SPLATINSERT20:%.*]] = insertelement <4 x i32> undef, i32 [[AMT1:%.*]], i32 0175; AVX2-NEXT: [[SPLAT2:%.*]] = shufflevector <4 x i32> [[SPLATINSERT20]], <4 x i32> undef, <4 x i32> zeroinitializer176; AVX2-NEXT: [[SPLATINSERT22:%.*]] = insertelement <4 x i32> undef, i32 [[X:%.*]], i32 0177; AVX2-NEXT: [[SPLAT3:%.*]] = shufflevector <4 x i32> [[SPLATINSERT22]], <4 x i32> undef, <4 x i32> zeroinitializer178; AVX2-NEXT: br label [[VECTOR_BODY:%.*]]179; AVX2: vector.body:180; AVX2-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]181; AVX2-NEXT: [[TMP0:%.*]] = getelementptr inbounds i8, ptr [[CONTROL:%.*]], i64 [[INDEX]]182; AVX2-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i8>, ptr [[TMP0]], align 1183; AVX2-NEXT: [[TMP2:%.*]] = icmp eq <4 x i8> [[WIDE_LOAD]], zeroinitializer184; AVX2-NEXT: [[TMP3:%.*]] = select <4 x i1> [[TMP2]], <4 x i32> [[SPLAT1]], <4 x i32> [[SPLAT2]]185; AVX2-NEXT: [[TMP4:%.*]] = shl <4 x i32> [[SPLAT3]], [[TMP3]]186; AVX2-NEXT: [[TMP5:%.*]] = getelementptr inbounds i32, ptr [[ARR:%.*]], i64 [[INDEX]]187; AVX2-NEXT: store <4 x i32> [[TMP4]], ptr [[TMP5]], align 4188; AVX2-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], 4189; AVX2-NEXT: [[TMP7:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]190; AVX2-NEXT: br i1 [[TMP7]], label [[EXIT]], label [[VECTOR_BODY]]191; AVX2: exit:192; AVX2-NEXT: ret void193;194; AVX512BW-LABEL: @vector_variable_shift_left_loop(195; AVX512BW-NEXT: entry:196; AVX512BW-NEXT: [[CMP16:%.*]] = icmp sgt i32 [[COUNT:%.*]], 0197; AVX512BW-NEXT: [[WIDE_TRIP_COUNT:%.*]] = zext i32 [[COUNT]] to i64198; AVX512BW-NEXT: br i1 [[CMP16]], label [[VECTOR_PH:%.*]], label [[EXIT:%.*]]199; AVX512BW: vector.ph:200; AVX512BW-NEXT: [[N_VEC:%.*]] = and i64 [[WIDE_TRIP_COUNT]], 4294967292201; AVX512BW-NEXT: [[SPLATINSERT18:%.*]] = insertelement <4 x i32> undef, i32 [[AMT0:%.*]], i32 0202; AVX512BW-NEXT: [[SPLAT1:%.*]] = shufflevector <4 x i32> [[SPLATINSERT18]], <4 x i32> undef, <4 x i32> zeroinitializer203; AVX512BW-NEXT: [[SPLATINSERT20:%.*]] = insertelement <4 x i32> undef, i32 [[AMT1:%.*]], i32 0204; AVX512BW-NEXT: [[SPLAT2:%.*]] = shufflevector <4 x i32> [[SPLATINSERT20]], <4 x i32> undef, <4 x i32> zeroinitializer205; AVX512BW-NEXT: [[SPLATINSERT22:%.*]] = insertelement <4 x i32> undef, i32 [[X:%.*]], i32 0206; AVX512BW-NEXT: [[SPLAT3:%.*]] = shufflevector <4 x i32> [[SPLATINSERT22]], <4 x i32> undef, <4 x i32> zeroinitializer207; AVX512BW-NEXT: br label [[VECTOR_BODY:%.*]]208; AVX512BW: vector.body:209; AVX512BW-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]210; AVX512BW-NEXT: [[TMP0:%.*]] = getelementptr inbounds i8, ptr [[CONTROL:%.*]], i64 [[INDEX]]211; AVX512BW-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i8>, ptr [[TMP0]], align 1212; AVX512BW-NEXT: [[TMP2:%.*]] = icmp eq <4 x i8> [[WIDE_LOAD]], zeroinitializer213; AVX512BW-NEXT: [[TMP3:%.*]] = select <4 x i1> [[TMP2]], <4 x i32> [[SPLAT1]], <4 x i32> [[SPLAT2]]214; AVX512BW-NEXT: [[TMP4:%.*]] = shl <4 x i32> [[SPLAT3]], [[TMP3]]215; AVX512BW-NEXT: [[TMP5:%.*]] = getelementptr inbounds i32, ptr [[ARR:%.*]], i64 [[INDEX]]216; AVX512BW-NEXT: store <4 x i32> [[TMP4]], ptr [[TMP5]], align 4217; AVX512BW-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], 4218; AVX512BW-NEXT: [[TMP7:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]219; AVX512BW-NEXT: br i1 [[TMP7]], label [[EXIT]], label [[VECTOR_BODY]]220; AVX512BW: exit:221; AVX512BW-NEXT: ret void222;223; XOP-LABEL: @vector_variable_shift_left_loop(224; XOP-NEXT: entry:225; XOP-NEXT: [[CMP16:%.*]] = icmp sgt i32 [[COUNT:%.*]], 0226; XOP-NEXT: [[WIDE_TRIP_COUNT:%.*]] = zext i32 [[COUNT]] to i64227; XOP-NEXT: br i1 [[CMP16]], label [[VECTOR_PH:%.*]], label [[EXIT:%.*]]228; XOP: vector.ph:229; XOP-NEXT: [[N_VEC:%.*]] = and i64 [[WIDE_TRIP_COUNT]], 4294967292230; XOP-NEXT: [[SPLATINSERT18:%.*]] = insertelement <4 x i32> undef, i32 [[AMT0:%.*]], i32 0231; XOP-NEXT: [[SPLAT1:%.*]] = shufflevector <4 x i32> [[SPLATINSERT18]], <4 x i32> undef, <4 x i32> zeroinitializer232; XOP-NEXT: [[SPLATINSERT20:%.*]] = insertelement <4 x i32> undef, i32 [[AMT1:%.*]], i32 0233; XOP-NEXT: [[SPLAT2:%.*]] = shufflevector <4 x i32> [[SPLATINSERT20]], <4 x i32> undef, <4 x i32> zeroinitializer234; XOP-NEXT: [[SPLATINSERT22:%.*]] = insertelement <4 x i32> undef, i32 [[X:%.*]], i32 0235; XOP-NEXT: [[SPLAT3:%.*]] = shufflevector <4 x i32> [[SPLATINSERT22]], <4 x i32> undef, <4 x i32> zeroinitializer236; XOP-NEXT: br label [[VECTOR_BODY:%.*]]237; XOP: vector.body:238; XOP-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]239; XOP-NEXT: [[TMP0:%.*]] = getelementptr inbounds i8, ptr [[CONTROL:%.*]], i64 [[INDEX]]240; XOP-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i8>, ptr [[TMP0]], align 1241; XOP-NEXT: [[TMP2:%.*]] = icmp eq <4 x i8> [[WIDE_LOAD]], zeroinitializer242; XOP-NEXT: [[TMP3:%.*]] = select <4 x i1> [[TMP2]], <4 x i32> [[SPLAT1]], <4 x i32> [[SPLAT2]]243; XOP-NEXT: [[TMP4:%.*]] = shl <4 x i32> [[SPLAT3]], [[TMP3]]244; XOP-NEXT: [[TMP5:%.*]] = getelementptr inbounds i32, ptr [[ARR:%.*]], i64 [[INDEX]]245; XOP-NEXT: store <4 x i32> [[TMP4]], ptr [[TMP5]], align 4246; XOP-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], 4247; XOP-NEXT: [[TMP7:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]248; XOP-NEXT: br i1 [[TMP7]], label [[EXIT]], label [[VECTOR_BODY]]249; XOP: exit:250; XOP-NEXT: ret void251;252entry:253 %cmp16 = icmp sgt i32 %count, 0254 %wide.trip.count = zext i32 %count to i64255 br i1 %cmp16, label %vector.ph, label %exit256 257vector.ph:258 %n.vec = and i64 %wide.trip.count, 4294967292259 %splatinsert18 = insertelement <4 x i32> undef, i32 %amt0, i32 0260 %splat1 = shufflevector <4 x i32> %splatinsert18, <4 x i32> undef, <4 x i32> zeroinitializer261 %splatinsert20 = insertelement <4 x i32> undef, i32 %amt1, i32 0262 %splat2 = shufflevector <4 x i32> %splatinsert20, <4 x i32> undef, <4 x i32> zeroinitializer263 %splatinsert22 = insertelement <4 x i32> undef, i32 %x, i32 0264 %splat3 = shufflevector <4 x i32> %splatinsert22, <4 x i32> undef, <4 x i32> zeroinitializer265 br label %vector.body266 267vector.body:268 %index = phi i64 [ 0, %vector.ph ], [ %index.next, %vector.body ]269 %0 = getelementptr inbounds i8, ptr %control, i64 %index270 %wide.load = load <4 x i8>, ptr %0, align 1271 %1 = icmp eq <4 x i8> %wide.load, zeroinitializer272 %2 = select <4 x i1> %1, <4 x i32> %splat1, <4 x i32> %splat2273 %3 = shl <4 x i32> %splat3, %2274 %4 = getelementptr inbounds i32, ptr %arr, i64 %index275 store <4 x i32> %3, ptr %4, align 4276 %index.next = add i64 %index, 4277 %5 = icmp eq i64 %index.next, %n.vec278 br i1 %5, label %exit, label %vector.body279 280exit:281 ret void282}283 284; PR37426 - https://bugs.llvm.org/show_bug.cgi?id=37426285; If we don't have real vector shift instructions (AVX1), convert the funnel286; shift into 2 funnel shifts and sink the splat shuffles into the loop.287 288define void @fancierRotate2(ptr %arr, ptr %control, i32 %rot0, i32 %rot1) {289; AVX1-LABEL: @fancierRotate2(290; AVX1-NEXT: entry:291; AVX1-NEXT: [[I0:%.*]] = insertelement <8 x i32> undef, i32 [[ROT0:%.*]], i32 0292; AVX1-NEXT: [[S0:%.*]] = shufflevector <8 x i32> [[I0]], <8 x i32> undef, <8 x i32> zeroinitializer293; AVX1-NEXT: [[I1:%.*]] = insertelement <8 x i32> undef, i32 [[ROT1:%.*]], i32 0294; AVX1-NEXT: [[S1:%.*]] = shufflevector <8 x i32> [[I1]], <8 x i32> undef, <8 x i32> zeroinitializer295; AVX1-NEXT: br label [[LOOP:%.*]]296; AVX1: loop:297; AVX1-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ], [ [[INDEX_NEXT:%.*]], [[LOOP]] ]298; AVX1-NEXT: [[T0:%.*]] = getelementptr inbounds i8, ptr [[CONTROL:%.*]], i64 [[INDEX]]299; AVX1-NEXT: [[WIDE_LOAD:%.*]] = load <8 x i8>, ptr [[T0]], align 1300; AVX1-NEXT: [[T2:%.*]] = icmp eq <8 x i8> [[WIDE_LOAD]], zeroinitializer301; AVX1-NEXT: [[SHAMT:%.*]] = select <8 x i1> [[T2]], <8 x i32> [[S0]], <8 x i32> [[S1]]302; AVX1-NEXT: [[T4:%.*]] = getelementptr inbounds i32, ptr [[ARR:%.*]], i64 [[INDEX]]303; AVX1-NEXT: [[WIDE_LOAD21:%.*]] = load <8 x i32>, ptr [[T4]], align 4304; AVX1-NEXT: [[TMP0:%.*]] = shufflevector <8 x i32> [[I0]], <8 x i32> undef, <8 x i32> zeroinitializer305; AVX1-NEXT: [[TMP1:%.*]] = call <8 x i32> @llvm.fshl.v8i32(<8 x i32> [[WIDE_LOAD21]], <8 x i32> [[WIDE_LOAD21]], <8 x i32> [[TMP0]])306; AVX1-NEXT: [[TMP2:%.*]] = shufflevector <8 x i32> [[I1]], <8 x i32> undef, <8 x i32> zeroinitializer307; AVX1-NEXT: [[TMP3:%.*]] = call <8 x i32> @llvm.fshl.v8i32(<8 x i32> [[WIDE_LOAD21]], <8 x i32> [[WIDE_LOAD21]], <8 x i32> [[TMP2]])308; AVX1-NEXT: [[TMP4:%.*]] = select <8 x i1> [[T2]], <8 x i32> [[TMP1]], <8 x i32> [[TMP3]]309; AVX1-NEXT: store <8 x i32> [[TMP4]], ptr [[T4]], align 4310; AVX1-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], 8311; AVX1-NEXT: [[T7:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024312; AVX1-NEXT: br i1 [[T7]], label [[EXIT:%.*]], label [[LOOP]]313; AVX1: exit:314; AVX1-NEXT: ret void315;316; AVX2-LABEL: @fancierRotate2(317; AVX2-NEXT: entry:318; AVX2-NEXT: [[I0:%.*]] = insertelement <8 x i32> undef, i32 [[ROT0:%.*]], i32 0319; AVX2-NEXT: [[S0:%.*]] = shufflevector <8 x i32> [[I0]], <8 x i32> undef, <8 x i32> zeroinitializer320; AVX2-NEXT: [[I1:%.*]] = insertelement <8 x i32> undef, i32 [[ROT1:%.*]], i32 0321; AVX2-NEXT: [[S1:%.*]] = shufflevector <8 x i32> [[I1]], <8 x i32> undef, <8 x i32> zeroinitializer322; AVX2-NEXT: br label [[LOOP:%.*]]323; AVX2: loop:324; AVX2-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ], [ [[INDEX_NEXT:%.*]], [[LOOP]] ]325; AVX2-NEXT: [[T0:%.*]] = getelementptr inbounds i8, ptr [[CONTROL:%.*]], i64 [[INDEX]]326; AVX2-NEXT: [[WIDE_LOAD:%.*]] = load <8 x i8>, ptr [[T0]], align 1327; AVX2-NEXT: [[T2:%.*]] = icmp eq <8 x i8> [[WIDE_LOAD]], zeroinitializer328; AVX2-NEXT: [[SHAMT:%.*]] = select <8 x i1> [[T2]], <8 x i32> [[S0]], <8 x i32> [[S1]]329; AVX2-NEXT: [[T4:%.*]] = getelementptr inbounds i32, ptr [[ARR:%.*]], i64 [[INDEX]]330; AVX2-NEXT: [[WIDE_LOAD21:%.*]] = load <8 x i32>, ptr [[T4]], align 4331; AVX2-NEXT: [[ROT:%.*]] = call <8 x i32> @llvm.fshl.v8i32(<8 x i32> [[WIDE_LOAD21]], <8 x i32> [[WIDE_LOAD21]], <8 x i32> [[SHAMT]])332; AVX2-NEXT: store <8 x i32> [[ROT]], ptr [[T4]], align 4333; AVX2-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], 8334; AVX2-NEXT: [[T7:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024335; AVX2-NEXT: br i1 [[T7]], label [[EXIT:%.*]], label [[LOOP]]336; AVX2: exit:337; AVX2-NEXT: ret void338;339; AVX512BW-LABEL: @fancierRotate2(340; AVX512BW-NEXT: entry:341; AVX512BW-NEXT: [[I0:%.*]] = insertelement <8 x i32> undef, i32 [[ROT0:%.*]], i32 0342; AVX512BW-NEXT: [[S0:%.*]] = shufflevector <8 x i32> [[I0]], <8 x i32> undef, <8 x i32> zeroinitializer343; AVX512BW-NEXT: [[I1:%.*]] = insertelement <8 x i32> undef, i32 [[ROT1:%.*]], i32 0344; AVX512BW-NEXT: [[S1:%.*]] = shufflevector <8 x i32> [[I1]], <8 x i32> undef, <8 x i32> zeroinitializer345; AVX512BW-NEXT: br label [[LOOP:%.*]]346; AVX512BW: loop:347; AVX512BW-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ], [ [[INDEX_NEXT:%.*]], [[LOOP]] ]348; AVX512BW-NEXT: [[T0:%.*]] = getelementptr inbounds i8, ptr [[CONTROL:%.*]], i64 [[INDEX]]349; AVX512BW-NEXT: [[WIDE_LOAD:%.*]] = load <8 x i8>, ptr [[T0]], align 1350; AVX512BW-NEXT: [[T2:%.*]] = icmp eq <8 x i8> [[WIDE_LOAD]], zeroinitializer351; AVX512BW-NEXT: [[SHAMT:%.*]] = select <8 x i1> [[T2]], <8 x i32> [[S0]], <8 x i32> [[S1]]352; AVX512BW-NEXT: [[T4:%.*]] = getelementptr inbounds i32, ptr [[ARR:%.*]], i64 [[INDEX]]353; AVX512BW-NEXT: [[WIDE_LOAD21:%.*]] = load <8 x i32>, ptr [[T4]], align 4354; AVX512BW-NEXT: [[ROT:%.*]] = call <8 x i32> @llvm.fshl.v8i32(<8 x i32> [[WIDE_LOAD21]], <8 x i32> [[WIDE_LOAD21]], <8 x i32> [[SHAMT]])355; AVX512BW-NEXT: store <8 x i32> [[ROT]], ptr [[T4]], align 4356; AVX512BW-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], 8357; AVX512BW-NEXT: [[T7:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024358; AVX512BW-NEXT: br i1 [[T7]], label [[EXIT:%.*]], label [[LOOP]]359; AVX512BW: exit:360; AVX512BW-NEXT: ret void361;362; XOP-LABEL: @fancierRotate2(363; XOP-NEXT: entry:364; XOP-NEXT: [[I0:%.*]] = insertelement <8 x i32> undef, i32 [[ROT0:%.*]], i32 0365; XOP-NEXT: [[S0:%.*]] = shufflevector <8 x i32> [[I0]], <8 x i32> undef, <8 x i32> zeroinitializer366; XOP-NEXT: [[I1:%.*]] = insertelement <8 x i32> undef, i32 [[ROT1:%.*]], i32 0367; XOP-NEXT: [[S1:%.*]] = shufflevector <8 x i32> [[I1]], <8 x i32> undef, <8 x i32> zeroinitializer368; XOP-NEXT: br label [[LOOP:%.*]]369; XOP: loop:370; XOP-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ], [ [[INDEX_NEXT:%.*]], [[LOOP]] ]371; XOP-NEXT: [[T0:%.*]] = getelementptr inbounds i8, ptr [[CONTROL:%.*]], i64 [[INDEX]]372; XOP-NEXT: [[WIDE_LOAD:%.*]] = load <8 x i8>, ptr [[T0]], align 1373; XOP-NEXT: [[T2:%.*]] = icmp eq <8 x i8> [[WIDE_LOAD]], zeroinitializer374; XOP-NEXT: [[SHAMT:%.*]] = select <8 x i1> [[T2]], <8 x i32> [[S0]], <8 x i32> [[S1]]375; XOP-NEXT: [[T4:%.*]] = getelementptr inbounds i32, ptr [[ARR:%.*]], i64 [[INDEX]]376; XOP-NEXT: [[WIDE_LOAD21:%.*]] = load <8 x i32>, ptr [[T4]], align 4377; XOP-NEXT: [[ROT:%.*]] = call <8 x i32> @llvm.fshl.v8i32(<8 x i32> [[WIDE_LOAD21]], <8 x i32> [[WIDE_LOAD21]], <8 x i32> [[SHAMT]])378; XOP-NEXT: store <8 x i32> [[ROT]], ptr [[T4]], align 4379; XOP-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], 8380; XOP-NEXT: [[T7:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024381; XOP-NEXT: br i1 [[T7]], label [[EXIT:%.*]], label [[LOOP]]382; XOP: exit:383; XOP-NEXT: ret void384;385entry:386 %i0 = insertelement <8 x i32> undef, i32 %rot0, i32 0387 %s0 = shufflevector <8 x i32> %i0, <8 x i32> undef, <8 x i32> zeroinitializer388 %i1 = insertelement <8 x i32> undef, i32 %rot1, i32 0389 %s1 = shufflevector <8 x i32> %i1, <8 x i32> undef, <8 x i32> zeroinitializer390 br label %loop391 392loop:393 %index = phi i64 [ 0, %entry ], [ %index.next, %loop ]394 %t0 = getelementptr inbounds i8, ptr %control, i64 %index395 %wide.load = load <8 x i8>, ptr %t0, align 1396 %t2 = icmp eq <8 x i8> %wide.load, zeroinitializer397 %shamt = select <8 x i1> %t2, <8 x i32> %s0, <8 x i32> %s1398 %t4 = getelementptr inbounds i32, ptr %arr, i64 %index399 %wide.load21 = load <8 x i32>, ptr %t4, align 4400 %rot = call <8 x i32> @llvm.fshl.v8i32(<8 x i32> %wide.load21, <8 x i32> %wide.load21, <8 x i32> %shamt)401 store <8 x i32> %rot, ptr %t4, align 4402 %index.next = add i64 %index, 8403 %t7 = icmp eq i64 %index.next, 1024404 br i1 %t7, label %exit, label %loop405 406exit:407 ret void408}409 410declare <8 x i32> @llvm.fshl.v8i32(<8 x i32>, <8 x i32>, <8 x i32>) #1411 412; Check that every instruction inserted by -passes='require<profile-summary>,function(codegenprepare)' has a debug location.413; DEBUG: CheckModuleDebugify: PASS414