227 lines · plain
1; NOTE: Assertions have been autogenerated by utils/update_test_checks.py2; RUN: opt -S %s -passes=scalarize-masked-mem-intrin -mtriple=aarch64-linux-gnu | FileCheck -check-prefixes=CHECK-LE-COMMON,CHECK-LE %s3; RUN: opt -S %s -passes=scalarize-masked-mem-intrin -mtriple=aarch64-linux-gnu -mattr=+sve | FileCheck -check-prefixes=CHECK-LE-COMMON,CHECK-LE-SVE %s4; RUN: opt -S %s -passes=scalarize-masked-mem-intrin -mtriple=aarch64_be-linux-gnu -data-layout="E-m:o-i64:64-i128:128-n32:64-S128" | FileCheck -check-prefixes=CHECK-BE %s5 6define <2 x i64> @scalarize_v2i64(ptr %p, <2 x i1> %mask, <2 x i64> %passthru) {7; CHECK-LE-LABEL: @scalarize_v2i64(8; CHECK-LE-NEXT: [[SCALAR_MASK:%.*]] = bitcast <2 x i1> [[MASK:%.*]] to i29; CHECK-LE-NEXT: [[TMP1:%.*]] = and i2 [[SCALAR_MASK]], 110; CHECK-LE-NEXT: [[TMP2:%.*]] = icmp ne i2 [[TMP1]], 011; CHECK-LE-NEXT: br i1 [[TMP2]], label [[COND_LOAD:%.*]], label [[ELSE:%.*]]12; CHECK-LE: cond.load:13; CHECK-LE-NEXT: [[TMP3:%.*]] = getelementptr inbounds i64, ptr [[P:%.*]], i32 014; CHECK-LE-NEXT: [[TMP4:%.*]] = load i64, ptr [[TMP3]], align 815; CHECK-LE-NEXT: [[TMP5:%.*]] = insertelement <2 x i64> [[PASSTHRU:%.*]], i64 [[TMP4]], i64 016; CHECK-LE-NEXT: br label [[ELSE]]17; CHECK-LE: else:18; CHECK-LE-NEXT: [[RES_PHI_ELSE:%.*]] = phi <2 x i64> [ [[TMP5]], [[COND_LOAD]] ], [ [[PASSTHRU]], [[TMP0:%.*]] ]19; CHECK-LE-NEXT: [[TMP6:%.*]] = and i2 [[SCALAR_MASK]], -220; CHECK-LE-NEXT: [[TMP7:%.*]] = icmp ne i2 [[TMP6]], 021; CHECK-LE-NEXT: br i1 [[TMP7]], label [[COND_LOAD1:%.*]], label [[ELSE2:%.*]]22; CHECK-LE: cond.load1:23; CHECK-LE-NEXT: [[TMP8:%.*]] = getelementptr inbounds i64, ptr [[P]], i32 124; CHECK-LE-NEXT: [[TMP9:%.*]] = load i64, ptr [[TMP8]], align 825; CHECK-LE-NEXT: [[TMP10:%.*]] = insertelement <2 x i64> [[RES_PHI_ELSE]], i64 [[TMP9]], i64 126; CHECK-LE-NEXT: br label [[ELSE2]]27; CHECK-LE: else2:28; CHECK-LE-NEXT: [[RES_PHI_ELSE3:%.*]] = phi <2 x i64> [ [[TMP10]], [[COND_LOAD1]] ], [ [[RES_PHI_ELSE]], [[ELSE]] ]29; CHECK-LE-NEXT: ret <2 x i64> [[RES_PHI_ELSE3]]30;31; CHECK-LE-SVE-LABEL: @scalarize_v2i64(32; CHECK-LE-SVE-NEXT: [[RET:%.*]] = call <2 x i64> @llvm.masked.load.v2i64.p0(ptr align 128 [[P:%.*]], <2 x i1> [[MASK:%.*]], <2 x i64> [[PASSTHRU:%.*]])33; CHECK-LE-SVE-NEXT: ret <2 x i64> [[RET]]34;35; CHECK-BE-LABEL: @scalarize_v2i64(36; CHECK-BE-NEXT: [[SCALAR_MASK:%.*]] = bitcast <2 x i1> [[MASK:%.*]] to i237; CHECK-BE-NEXT: [[TMP1:%.*]] = and i2 [[SCALAR_MASK]], -238; CHECK-BE-NEXT: [[TMP2:%.*]] = icmp ne i2 [[TMP1]], 039; CHECK-BE-NEXT: br i1 [[TMP2]], label [[COND_LOAD:%.*]], label [[ELSE:%.*]]40; CHECK-BE: cond.load:41; CHECK-BE-NEXT: [[TMP3:%.*]] = getelementptr inbounds i64, ptr [[P:%.*]], i32 042; CHECK-BE-NEXT: [[TMP4:%.*]] = load i64, ptr [[TMP3]], align 843; CHECK-BE-NEXT: [[TMP5:%.*]] = insertelement <2 x i64> [[PASSTHRU:%.*]], i64 [[TMP4]], i64 044; CHECK-BE-NEXT: br label [[ELSE]]45; CHECK-BE: else:46; CHECK-BE-NEXT: [[RES_PHI_ELSE:%.*]] = phi <2 x i64> [ [[TMP5]], [[COND_LOAD]] ], [ [[PASSTHRU]], [[TMP0:%.*]] ]47; CHECK-BE-NEXT: [[TMP6:%.*]] = and i2 [[SCALAR_MASK]], 148; CHECK-BE-NEXT: [[TMP7:%.*]] = icmp ne i2 [[TMP6]], 049; CHECK-BE-NEXT: br i1 [[TMP7]], label [[COND_LOAD1:%.*]], label [[ELSE2:%.*]]50; CHECK-BE: cond.load1:51; CHECK-BE-NEXT: [[TMP8:%.*]] = getelementptr inbounds i64, ptr [[P]], i32 152; CHECK-BE-NEXT: [[TMP9:%.*]] = load i64, ptr [[TMP8]], align 853; CHECK-BE-NEXT: [[TMP10:%.*]] = insertelement <2 x i64> [[RES_PHI_ELSE]], i64 [[TMP9]], i64 154; CHECK-BE-NEXT: br label [[ELSE2]]55; CHECK-BE: else2:56; CHECK-BE-NEXT: [[RES_PHI_ELSE3:%.*]] = phi <2 x i64> [ [[TMP10]], [[COND_LOAD1]] ], [ [[RES_PHI_ELSE]], [[ELSE]] ]57; CHECK-BE-NEXT: ret <2 x i64> [[RES_PHI_ELSE3]]58;59 %ret = call <2 x i64> @llvm.masked.load.v2i64.p0(ptr %p, i32 128, <2 x i1> %mask, <2 x i64> %passthru)60 ret <2 x i64> %ret61}62 63define <2 x i64> @scalarize_v2i64_ones_mask(ptr %p, <2 x i64> %passthru) {64; CHECK-LE-LABEL: @scalarize_v2i64_ones_mask(65; CHECK-LE-NEXT: [[TMP1:%.*]] = load <2 x i64>, ptr [[P:%.*]], align 866; CHECK-LE-NEXT: ret <2 x i64> [[TMP1]]67;68; CHECK-LE-SVE-LABEL: @scalarize_v2i64_ones_mask(69; CHECK-LE-SVE-NEXT: [[RET:%.*]] = call <2 x i64> @llvm.masked.load.v2i64.p0(ptr align 8 [[P:%.*]], <2 x i1> splat (i1 true), <2 x i64> [[PASSTHRU:%.*]])70; CHECK-LE-SVE-NEXT: ret <2 x i64> [[RET]]71;72; CHECK-BE-LABEL: @scalarize_v2i64_ones_mask(73; CHECK-BE-NEXT: [[TMP1:%.*]] = load <2 x i64>, ptr [[P:%.*]], align 874; CHECK-BE-NEXT: ret <2 x i64> [[TMP1]]75;76 %ret = call <2 x i64> @llvm.masked.load.v2i64.p0(ptr %p, i32 8, <2 x i1> <i1 true, i1 true>, <2 x i64> %passthru)77 ret <2 x i64> %ret78}79 80define <2 x i64> @scalarize_v2i64_zero_mask(ptr %p, <2 x i64> %passthru) {81; CHECK-LE-LABEL: @scalarize_v2i64_zero_mask(82; CHECK-LE-NEXT: ret <2 x i64> [[PASSTHRU:%.*]]83;84; CHECK-LE-SVE-LABEL: @scalarize_v2i64_zero_mask(85; CHECK-LE-SVE-NEXT: [[RET:%.*]] = call <2 x i64> @llvm.masked.load.v2i64.p0(ptr align 8 [[P:%.*]], <2 x i1> zeroinitializer, <2 x i64> [[PASSTHRU:%.*]])86; CHECK-LE-SVE-NEXT: ret <2 x i64> [[RET]]87;88; CHECK-BE-LABEL: @scalarize_v2i64_zero_mask(89; CHECK-BE-NEXT: ret <2 x i64> [[PASSTHRU:%.*]]90;91 %ret = call <2 x i64> @llvm.masked.load.v2i64.p0(ptr %p, i32 8, <2 x i1> <i1 false, i1 false>, <2 x i64> %passthru)92 ret <2 x i64> %ret93}94 95define <2 x i64> @scalarize_v2i64_const_mask(ptr %p, <2 x i64> %passthru) {96; CHECK-LE-LABEL: @scalarize_v2i64_const_mask(97; CHECK-LE-NEXT: [[TMP1:%.*]] = getelementptr inbounds i64, ptr [[P:%.*]], i32 198; CHECK-LE-NEXT: [[TMP2:%.*]] = load i64, ptr [[TMP1]], align 899; CHECK-LE-NEXT: [[TMP3:%.*]] = insertelement <2 x i64> [[PASSTHRU:%.*]], i64 [[TMP2]], i64 1100; CHECK-LE-NEXT: ret <2 x i64> [[TMP3]]101;102; CHECK-LE-SVE-LABEL: @scalarize_v2i64_const_mask(103; CHECK-LE-SVE-NEXT: [[RET:%.*]] = call <2 x i64> @llvm.masked.load.v2i64.p0(ptr align 8 [[P:%.*]], <2 x i1> <i1 false, i1 true>, <2 x i64> [[PASSTHRU:%.*]])104; CHECK-LE-SVE-NEXT: ret <2 x i64> [[RET]]105;106; CHECK-BE-LABEL: @scalarize_v2i64_const_mask(107; CHECK-BE-NEXT: [[TMP1:%.*]] = getelementptr inbounds i64, ptr [[P:%.*]], i32 1108; CHECK-BE-NEXT: [[TMP2:%.*]] = load i64, ptr [[TMP1]], align 8109; CHECK-BE-NEXT: [[TMP3:%.*]] = insertelement <2 x i64> [[PASSTHRU:%.*]], i64 [[TMP2]], i64 1110; CHECK-BE-NEXT: ret <2 x i64> [[TMP3]]111;112 %ret = call <2 x i64> @llvm.masked.load.v2i64.p0(ptr %p, i32 8, <2 x i1> <i1 false, i1 true>, <2 x i64> %passthru)113 ret <2 x i64> %ret114}115 116; This use a byte sized but non power of 2 element size. This used to crash due to bad alignment calculation.117define <2 x i24> @scalarize_v2i24(ptr %p, <2 x i1> %mask, <2 x i24> %passthru) {118; CHECK-LE-COMMON-LABEL: @scalarize_v2i24(119; CHECK-LE-COMMON-NEXT: [[SCALAR_MASK:%.*]] = bitcast <2 x i1> [[MASK:%.*]] to i2120; CHECK-LE-COMMON-NEXT: [[TMP1:%.*]] = and i2 [[SCALAR_MASK]], 1121; CHECK-LE-COMMON-NEXT: [[TMP2:%.*]] = icmp ne i2 [[TMP1]], 0122; CHECK-LE-COMMON-NEXT: br i1 [[TMP2]], label [[COND_LOAD:%.*]], label [[ELSE:%.*]]123; CHECK-LE-COMMON: cond.load:124; CHECK-LE-COMMON-NEXT: [[TMP3:%.*]] = getelementptr inbounds i24, ptr [[P:%.*]], i32 0125; CHECK-LE-COMMON-NEXT: [[TMP4:%.*]] = load i24, ptr [[TMP3]], align 1126; CHECK-LE-COMMON-NEXT: [[TMP5:%.*]] = insertelement <2 x i24> [[PASSTHRU:%.*]], i24 [[TMP4]], i64 0127; CHECK-LE-COMMON-NEXT: br label [[ELSE]]128; CHECK-LE-COMMON: else:129; CHECK-LE-COMMON-NEXT: [[RES_PHI_ELSE:%.*]] = phi <2 x i24> [ [[TMP5]], [[COND_LOAD]] ], [ [[PASSTHRU]], [[TMP0:%.*]] ]130; CHECK-LE-COMMON-NEXT: [[TMP6:%.*]] = and i2 [[SCALAR_MASK]], -2131; CHECK-LE-COMMON-NEXT: [[TMP7:%.*]] = icmp ne i2 [[TMP6]], 0132; CHECK-LE-COMMON-NEXT: br i1 [[TMP7]], label [[COND_LOAD1:%.*]], label [[ELSE2:%.*]]133; CHECK-LE-COMMON: cond.load1:134; CHECK-LE-COMMON-NEXT: [[TMP8:%.*]] = getelementptr inbounds i24, ptr [[P]], i32 1135; CHECK-LE-COMMON-NEXT: [[TMP9:%.*]] = load i24, ptr [[TMP8]], align 1136; CHECK-LE-COMMON-NEXT: [[TMP10:%.*]] = insertelement <2 x i24> [[RES_PHI_ELSE]], i24 [[TMP9]], i64 1137; CHECK-LE-COMMON-NEXT: br label [[ELSE2]]138; CHECK-LE-COMMON: else2:139; CHECK-LE-COMMON-NEXT: [[RES_PHI_ELSE3:%.*]] = phi <2 x i24> [ [[TMP10]], [[COND_LOAD1]] ], [ [[RES_PHI_ELSE]], [[ELSE]] ]140; CHECK-LE-COMMON-NEXT: ret <2 x i24> [[RES_PHI_ELSE3]]141;142; CHECK-BE-LABEL: @scalarize_v2i24(143; CHECK-BE-NEXT: [[SCALAR_MASK:%.*]] = bitcast <2 x i1> [[MASK:%.*]] to i2144; CHECK-BE-NEXT: [[TMP1:%.*]] = and i2 [[SCALAR_MASK]], -2145; CHECK-BE-NEXT: [[TMP2:%.*]] = icmp ne i2 [[TMP1]], 0146; CHECK-BE-NEXT: br i1 [[TMP2]], label [[COND_LOAD:%.*]], label [[ELSE:%.*]]147; CHECK-BE: cond.load:148; CHECK-BE-NEXT: [[TMP3:%.*]] = getelementptr inbounds i24, ptr [[P:%.*]], i32 0149; CHECK-BE-NEXT: [[TMP4:%.*]] = load i24, ptr [[TMP3]], align 1150; CHECK-BE-NEXT: [[TMP5:%.*]] = insertelement <2 x i24> [[PASSTHRU:%.*]], i24 [[TMP4]], i64 0151; CHECK-BE-NEXT: br label [[ELSE]]152; CHECK-BE: else:153; CHECK-BE-NEXT: [[RES_PHI_ELSE:%.*]] = phi <2 x i24> [ [[TMP5]], [[COND_LOAD]] ], [ [[PASSTHRU]], [[TMP0:%.*]] ]154; CHECK-BE-NEXT: [[TMP6:%.*]] = and i2 [[SCALAR_MASK]], 1155; CHECK-BE-NEXT: [[TMP7:%.*]] = icmp ne i2 [[TMP6]], 0156; CHECK-BE-NEXT: br i1 [[TMP7]], label [[COND_LOAD1:%.*]], label [[ELSE2:%.*]]157; CHECK-BE: cond.load1:158; CHECK-BE-NEXT: [[TMP8:%.*]] = getelementptr inbounds i24, ptr [[P]], i32 1159; CHECK-BE-NEXT: [[TMP9:%.*]] = load i24, ptr [[TMP8]], align 1160; CHECK-BE-NEXT: [[TMP10:%.*]] = insertelement <2 x i24> [[RES_PHI_ELSE]], i24 [[TMP9]], i64 1161; CHECK-BE-NEXT: br label [[ELSE2]]162; CHECK-BE: else2:163; CHECK-BE-NEXT: [[RES_PHI_ELSE3:%.*]] = phi <2 x i24> [ [[TMP10]], [[COND_LOAD1]] ], [ [[RES_PHI_ELSE]], [[ELSE]] ]164; CHECK-BE-NEXT: ret <2 x i24> [[RES_PHI_ELSE3]]165;166 %ret = call <2 x i24> @llvm.masked.load.v2i24.p0(ptr %p, i32 8, <2 x i1> %mask, <2 x i24> %passthru)167 ret <2 x i24> %ret168}169 170; This use a byte sized but non power of 2 element size. This used to crash due to bad alignment calculation.171define <2 x i48> @scalarize_v2i48(ptr %p, <2 x i1> %mask, <2 x i48> %passthru) {172; CHECK-LE-COMMON-LABEL: @scalarize_v2i48(173; CHECK-LE-COMMON-NEXT: [[SCALAR_MASK:%.*]] = bitcast <2 x i1> [[MASK:%.*]] to i2174; CHECK-LE-COMMON-NEXT: [[TMP1:%.*]] = and i2 [[SCALAR_MASK]], 1175; CHECK-LE-COMMON-NEXT: [[TMP2:%.*]] = icmp ne i2 [[TMP1]], 0176; CHECK-LE-COMMON-NEXT: br i1 [[TMP2]], label [[COND_LOAD:%.*]], label [[ELSE:%.*]]177; CHECK-LE-COMMON: cond.load:178; CHECK-LE-COMMON-NEXT: [[TMP3:%.*]] = getelementptr inbounds i48, ptr [[P:%.*]], i32 0179; CHECK-LE-COMMON-NEXT: [[TMP4:%.*]] = load i48, ptr [[TMP3]], align 2180; CHECK-LE-COMMON-NEXT: [[TMP5:%.*]] = insertelement <2 x i48> [[PASSTHRU:%.*]], i48 [[TMP4]], i64 0181; CHECK-LE-COMMON-NEXT: br label [[ELSE]]182; CHECK-LE-COMMON: else:183; CHECK-LE-COMMON-NEXT: [[RES_PHI_ELSE:%.*]] = phi <2 x i48> [ [[TMP5]], [[COND_LOAD]] ], [ [[PASSTHRU]], [[TMP0:%.*]] ]184; CHECK-LE-COMMON-NEXT: [[TMP6:%.*]] = and i2 [[SCALAR_MASK]], -2185; CHECK-LE-COMMON-NEXT: [[TMP7:%.*]] = icmp ne i2 [[TMP6]], 0186; CHECK-LE-COMMON-NEXT: br i1 [[TMP7]], label [[COND_LOAD1:%.*]], label [[ELSE2:%.*]]187; CHECK-LE-COMMON: cond.load1:188; CHECK-LE-COMMON-NEXT: [[TMP8:%.*]] = getelementptr inbounds i48, ptr [[P]], i32 1189; CHECK-LE-COMMON-NEXT: [[TMP9:%.*]] = load i48, ptr [[TMP8]], align 2190; CHECK-LE-COMMON-NEXT: [[TMP10:%.*]] = insertelement <2 x i48> [[RES_PHI_ELSE]], i48 [[TMP9]], i64 1191; CHECK-LE-COMMON-NEXT: br label [[ELSE2]]192; CHECK-LE-COMMON: else2:193; CHECK-LE-COMMON-NEXT: [[RES_PHI_ELSE3:%.*]] = phi <2 x i48> [ [[TMP10]], [[COND_LOAD1]] ], [ [[RES_PHI_ELSE]], [[ELSE]] ]194; CHECK-LE-COMMON-NEXT: ret <2 x i48> [[RES_PHI_ELSE3]]195;196; CHECK-BE-LABEL: @scalarize_v2i48(197; CHECK-BE-NEXT: [[SCALAR_MASK:%.*]] = bitcast <2 x i1> [[MASK:%.*]] to i2198; CHECK-BE-NEXT: [[TMP1:%.*]] = and i2 [[SCALAR_MASK]], -2199; CHECK-BE-NEXT: [[TMP2:%.*]] = icmp ne i2 [[TMP1]], 0200; CHECK-BE-NEXT: br i1 [[TMP2]], label [[COND_LOAD:%.*]], label [[ELSE:%.*]]201; CHECK-BE: cond.load:202; CHECK-BE-NEXT: [[TMP3:%.*]] = getelementptr inbounds i48, ptr [[P:%.*]], i32 0203; CHECK-BE-NEXT: [[TMP4:%.*]] = load i48, ptr [[TMP3]], align 2204; CHECK-BE-NEXT: [[TMP5:%.*]] = insertelement <2 x i48> [[PASSTHRU:%.*]], i48 [[TMP4]], i64 0205; CHECK-BE-NEXT: br label [[ELSE]]206; CHECK-BE: else:207; CHECK-BE-NEXT: [[RES_PHI_ELSE:%.*]] = phi <2 x i48> [ [[TMP5]], [[COND_LOAD]] ], [ [[PASSTHRU]], [[TMP0:%.*]] ]208; CHECK-BE-NEXT: [[TMP6:%.*]] = and i2 [[SCALAR_MASK]], 1209; CHECK-BE-NEXT: [[TMP7:%.*]] = icmp ne i2 [[TMP6]], 0210; CHECK-BE-NEXT: br i1 [[TMP7]], label [[COND_LOAD1:%.*]], label [[ELSE2:%.*]]211; CHECK-BE: cond.load1:212; CHECK-BE-NEXT: [[TMP8:%.*]] = getelementptr inbounds i48, ptr [[P]], i32 1213; CHECK-BE-NEXT: [[TMP9:%.*]] = load i48, ptr [[TMP8]], align 2214; CHECK-BE-NEXT: [[TMP10:%.*]] = insertelement <2 x i48> [[RES_PHI_ELSE]], i48 [[TMP9]], i64 1215; CHECK-BE-NEXT: br label [[ELSE2]]216; CHECK-BE: else2:217; CHECK-BE-NEXT: [[RES_PHI_ELSE3:%.*]] = phi <2 x i48> [ [[TMP10]], [[COND_LOAD1]] ], [ [[RES_PHI_ELSE]], [[ELSE]] ]218; CHECK-BE-NEXT: ret <2 x i48> [[RES_PHI_ELSE3]]219;220 %ret = call <2 x i48> @llvm.masked.load.v2i48.p0(ptr %p, i32 16, <2 x i1> %mask, <2 x i48> %passthru)221 ret <2 x i48> %ret222}223 224declare <2 x i24> @llvm.masked.load.v2i24.p0(ptr, i32, <2 x i1>, <2 x i24>)225declare <2 x i48> @llvm.masked.load.v2i48.p0(ptr, i32, <2 x i1>, <2 x i48>)226declare <2 x i64> @llvm.masked.load.v2i64.p0(ptr, i32, <2 x i1>, <2 x i64>)227