463 lines · plain
1; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py UTC_ARGS: --version 22; Check memory cost model action for fixed vector SVE and Neon3; Vector bits size lower than 256 bits end up assuming Neon cost model4; CHECK-NEON has same performance as CHECK-SVE-1285 6; RUN: opt -passes="print<cost-model>" -cost-kind=all 2>&1 -disable-output -mtriple=aarch64--linux-gnu -mattr=+neon < %s | FileCheck %s --check-prefix=CHECK-NEON7; RUN: opt -passes="print<cost-model>" -cost-kind=all 2>&1 -disable-output -mtriple=aarch64--linux-gnu -mattr=+sve -aarch64-sve-vector-bits-min=128 < %s | FileCheck %s --check-prefix=CHECK-SVE-1288; RUN: opt -passes="print<cost-model>" -cost-kind=all 2>&1 -disable-output -mtriple=aarch64--linux-gnu -mattr=+sve -aarch64-sve-vector-bits-min=256 < %s | FileCheck %s --check-prefix=CHECK-SVE-2569; RUN: opt -passes="print<cost-model>" -cost-kind=all 2>&1 -disable-output -mtriple=aarch64--linux-gnu -mattr=+sve -aarch64-sve-vector-bits-min=512 < %s | FileCheck %s --check-prefix=CHECK-SVE-51210 11target datalayout = "e-m:e-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128"12 13define <16 x i8> @load16(ptr %ptr) {14; CHECK: function 'load16'15; CHECK-NEON-LABEL: 'load16'16; CHECK-NEON-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:4 SizeLat:1 for: %out = load <16 x i8>, ptr %ptr, align 1617; CHECK-NEON-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret <16 x i8> %out18;19; CHECK-SVE-128-LABEL: 'load16'20; CHECK-SVE-128-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:4 SizeLat:1 for: %out = load <16 x i8>, ptr %ptr, align 1621; CHECK-SVE-128-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret <16 x i8> %out22;23; CHECK-SVE-256-LABEL: 'load16'24; CHECK-SVE-256-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:4 SizeLat:1 for: %out = load <16 x i8>, ptr %ptr, align 1625; CHECK-SVE-256-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret <16 x i8> %out26;27; CHECK-SVE-512-LABEL: 'load16'28; CHECK-SVE-512-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:4 SizeLat:1 for: %out = load <16 x i8>, ptr %ptr, align 1629; CHECK-SVE-512-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret <16 x i8> %out30;31 %out = load <16 x i8>, ptr %ptr32 ret <16 x i8> %out33}34 35define void @store16(ptr %ptr, <16 x i8> %val) {36; CHECK: function 'store16'37; CHECK-NEON-LABEL: 'store16'38; CHECK-NEON-NEXT: Cost Model: Found costs of 1 for: store <16 x i8> %val, ptr %ptr, align 1639; CHECK-NEON-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void40;41; CHECK-SVE-128-LABEL: 'store16'42; CHECK-SVE-128-NEXT: Cost Model: Found costs of 1 for: store <16 x i8> %val, ptr %ptr, align 1643; CHECK-SVE-128-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void44;45; CHECK-SVE-256-LABEL: 'store16'46; CHECK-SVE-256-NEXT: Cost Model: Found costs of 1 for: store <16 x i8> %val, ptr %ptr, align 1647; CHECK-SVE-256-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void48;49; CHECK-SVE-512-LABEL: 'store16'50; CHECK-SVE-512-NEXT: Cost Model: Found costs of 1 for: store <16 x i8> %val, ptr %ptr, align 1651; CHECK-SVE-512-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void52;53 store <16 x i8> %val, ptr %ptr54 ret void55}56 57define <8 x i8> @load8(ptr %ptr) {58; CHECK: function 'load8'59; CHECK-NEON-LABEL: 'load8'60; CHECK-NEON-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:4 SizeLat:1 for: %out = load <8 x i8>, ptr %ptr, align 861; CHECK-NEON-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret <8 x i8> %out62;63; CHECK-SVE-128-LABEL: 'load8'64; CHECK-SVE-128-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:4 SizeLat:1 for: %out = load <8 x i8>, ptr %ptr, align 865; CHECK-SVE-128-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret <8 x i8> %out66;67; CHECK-SVE-256-LABEL: 'load8'68; CHECK-SVE-256-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:4 SizeLat:1 for: %out = load <8 x i8>, ptr %ptr, align 869; CHECK-SVE-256-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret <8 x i8> %out70;71; CHECK-SVE-512-LABEL: 'load8'72; CHECK-SVE-512-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:4 SizeLat:1 for: %out = load <8 x i8>, ptr %ptr, align 873; CHECK-SVE-512-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret <8 x i8> %out74;75 %out = load <8 x i8>, ptr %ptr76 ret <8 x i8> %out77}78 79define void @store8(ptr %ptr, <8 x i8> %val) {80; CHECK: function 'store8'81; CHECK-NEON-LABEL: 'store8'82; CHECK-NEON-NEXT: Cost Model: Found costs of 1 for: store <8 x i8> %val, ptr %ptr, align 883; CHECK-NEON-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void84;85; CHECK-SVE-128-LABEL: 'store8'86; CHECK-SVE-128-NEXT: Cost Model: Found costs of 1 for: store <8 x i8> %val, ptr %ptr, align 887; CHECK-SVE-128-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void88;89; CHECK-SVE-256-LABEL: 'store8'90; CHECK-SVE-256-NEXT: Cost Model: Found costs of 1 for: store <8 x i8> %val, ptr %ptr, align 891; CHECK-SVE-256-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void92;93; CHECK-SVE-512-LABEL: 'store8'94; CHECK-SVE-512-NEXT: Cost Model: Found costs of 1 for: store <8 x i8> %val, ptr %ptr, align 895; CHECK-SVE-512-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void96;97 store <8 x i8> %val, ptr %ptr98 ret void99}100 101define <4 x i8> @load4(ptr %ptr) {102; CHECK: function 'load4'103; CHECK-NEON-LABEL: 'load4'104; CHECK-NEON-NEXT: Cost Model: Found costs of RThru:2 CodeSize:1 Lat:4 SizeLat:1 for: %out = load <4 x i8>, ptr %ptr, align 4105; CHECK-NEON-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret <4 x i8> %out106;107; CHECK-SVE-128-LABEL: 'load4'108; CHECK-SVE-128-NEXT: Cost Model: Found costs of RThru:2 CodeSize:1 Lat:4 SizeLat:1 for: %out = load <4 x i8>, ptr %ptr, align 4109; CHECK-SVE-128-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret <4 x i8> %out110;111; CHECK-SVE-256-LABEL: 'load4'112; CHECK-SVE-256-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:4 SizeLat:1 for: %out = load <4 x i8>, ptr %ptr, align 4113; CHECK-SVE-256-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret <4 x i8> %out114;115; CHECK-SVE-512-LABEL: 'load4'116; CHECK-SVE-512-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:4 SizeLat:1 for: %out = load <4 x i8>, ptr %ptr, align 4117; CHECK-SVE-512-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret <4 x i8> %out118;119 %out = load <4 x i8>, ptr %ptr120 ret <4 x i8> %out121}122 123define void @store4(ptr %ptr, <4 x i8> %val) {124; CHECK: function 'store4'125; CHECK-NEON-LABEL: 'store4'126; CHECK-NEON-NEXT: Cost Model: Found costs of RThru:2 CodeSize:1 Lat:1 SizeLat:1 for: store <4 x i8> %val, ptr %ptr, align 4127; CHECK-NEON-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void128;129; CHECK-SVE-128-LABEL: 'store4'130; CHECK-SVE-128-NEXT: Cost Model: Found costs of RThru:2 CodeSize:1 Lat:1 SizeLat:1 for: store <4 x i8> %val, ptr %ptr, align 4131; CHECK-SVE-128-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void132;133; CHECK-SVE-256-LABEL: 'store4'134; CHECK-SVE-256-NEXT: Cost Model: Found costs of 1 for: store <4 x i8> %val, ptr %ptr, align 4135; CHECK-SVE-256-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void136;137; CHECK-SVE-512-LABEL: 'store4'138; CHECK-SVE-512-NEXT: Cost Model: Found costs of 1 for: store <4 x i8> %val, ptr %ptr, align 4139; CHECK-SVE-512-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void140;141 store <4 x i8> %val, ptr %ptr142 ret void143}144 145define <16 x i16> @load_256(ptr %ptr) {146; CHECK: function 'load_256'147; CHECK-NEON-LABEL: 'load_256'148; CHECK-NEON-NEXT: Cost Model: Found costs of RThru:2 CodeSize:2 Lat:4 SizeLat:2 for: %out = load <16 x i16>, ptr %ptr, align 32149; CHECK-NEON-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret <16 x i16> %out150;151; CHECK-SVE-128-LABEL: 'load_256'152; CHECK-SVE-128-NEXT: Cost Model: Found costs of RThru:2 CodeSize:2 Lat:4 SizeLat:2 for: %out = load <16 x i16>, ptr %ptr, align 32153; CHECK-SVE-128-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret <16 x i16> %out154;155; CHECK-SVE-256-LABEL: 'load_256'156; CHECK-SVE-256-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:4 SizeLat:1 for: %out = load <16 x i16>, ptr %ptr, align 32157; CHECK-SVE-256-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret <16 x i16> %out158;159; CHECK-SVE-512-LABEL: 'load_256'160; CHECK-SVE-512-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:4 SizeLat:1 for: %out = load <16 x i16>, ptr %ptr, align 32161; CHECK-SVE-512-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret <16 x i16> %out162;163 %out = load <16 x i16>, ptr %ptr164 ret <16 x i16> %out165}166 167define <8 x i64> @load_512(ptr %ptr) {168; CHECK: function 'load_512'169; CHECK-NEON-LABEL: 'load_512'170; CHECK-NEON-NEXT: Cost Model: Found costs of 4 for: %out = load <8 x i64>, ptr %ptr, align 64171; CHECK-NEON-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret <8 x i64> %out172;173; CHECK-SVE-128-LABEL: 'load_512'174; CHECK-SVE-128-NEXT: Cost Model: Found costs of 4 for: %out = load <8 x i64>, ptr %ptr, align 64175; CHECK-SVE-128-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret <8 x i64> %out176;177; CHECK-SVE-256-LABEL: 'load_512'178; CHECK-SVE-256-NEXT: Cost Model: Found costs of RThru:2 CodeSize:2 Lat:4 SizeLat:2 for: %out = load <8 x i64>, ptr %ptr, align 64179; CHECK-SVE-256-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret <8 x i64> %out180;181; CHECK-SVE-512-LABEL: 'load_512'182; CHECK-SVE-512-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:4 SizeLat:1 for: %out = load <8 x i64>, ptr %ptr, align 64183; CHECK-SVE-512-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret <8 x i64> %out184;185 %out = load <8 x i64>, ptr %ptr186 ret <8 x i64> %out187}188 189declare <4 x i8> @llvm.masked.gather.v4i8.v4p0(<4 x ptr>, i32 immarg, <4 x i1>, <4 x i8>)190define <4 x i8> @gather_load_4xi8_constant_mask(<4 x ptr> %ptrs) {191; CHECK: gather_load_4xi8_constant_mask192; CHECK-NEON-LABEL: 'gather_load_4xi8_constant_mask'193; CHECK-NEON-NEXT: Cost Model: Found costs of RThru:20 CodeSize:12 Lat:20 SizeLat:20 for: %lv = call <4 x i8> @llvm.masked.gather.v4i8.v4p0(<4 x ptr> align 1 %ptrs, <4 x i1> splat (i1 true), <4 x i8> undef)194; CHECK-NEON-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret <4 x i8> %lv195;196; CHECK-SVE-128-LABEL: 'gather_load_4xi8_constant_mask'197; CHECK-SVE-128-NEXT: Cost Model: Found costs of RThru:20 CodeSize:12 Lat:20 SizeLat:20 for: %lv = call <4 x i8> @llvm.masked.gather.v4i8.v4p0(<4 x ptr> align 1 %ptrs, <4 x i1> splat (i1 true), <4 x i8> undef)198; CHECK-SVE-128-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret <4 x i8> %lv199;200; CHECK-SVE-256-LABEL: 'gather_load_4xi8_constant_mask'201; CHECK-SVE-256-NEXT: Cost Model: Found costs of 40 for: %lv = call <4 x i8> @llvm.masked.gather.v4i8.v4p0(<4 x ptr> align 1 %ptrs, <4 x i1> splat (i1 true), <4 x i8> undef)202; CHECK-SVE-256-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret <4 x i8> %lv203;204; CHECK-SVE-512-LABEL: 'gather_load_4xi8_constant_mask'205; CHECK-SVE-512-NEXT: Cost Model: Found costs of 40 for: %lv = call <4 x i8> @llvm.masked.gather.v4i8.v4p0(<4 x ptr> align 1 %ptrs, <4 x i1> splat (i1 true), <4 x i8> undef)206; CHECK-SVE-512-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret <4 x i8> %lv207;208 %lv = call <4 x i8> @llvm.masked.gather.v4i8.v4p0(<4 x ptr> %ptrs, i32 1, <4 x i1> <i1 true, i1 true, i1 true, i1 true>, <4 x i8> undef)209 ret <4 x i8> %lv210}211 212define <4 x i8> @gather_load_4xi8_variable_mask(<4 x ptr> %ptrs, <4 x i1> %cond) {213; CHECK: gather_load_4xi8_variable_mask214; CHECK-NEON-LABEL: 'gather_load_4xi8_variable_mask'215; CHECK-NEON-NEXT: Cost Model: Found costs of RThru:28 CodeSize:20 Lat:32 SizeLat:32 for: %lv = call <4 x i8> @llvm.masked.gather.v4i8.v4p0(<4 x ptr> align 1 %ptrs, <4 x i1> %cond, <4 x i8> undef)216; CHECK-NEON-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret <4 x i8> %lv217;218; CHECK-SVE-128-LABEL: 'gather_load_4xi8_variable_mask'219; CHECK-SVE-128-NEXT: Cost Model: Found costs of RThru:28 CodeSize:20 Lat:32 SizeLat:32 for: %lv = call <4 x i8> @llvm.masked.gather.v4i8.v4p0(<4 x ptr> align 1 %ptrs, <4 x i1> %cond, <4 x i8> undef)220; CHECK-SVE-128-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret <4 x i8> %lv221;222; CHECK-SVE-256-LABEL: 'gather_load_4xi8_variable_mask'223; CHECK-SVE-256-NEXT: Cost Model: Found costs of 40 for: %lv = call <4 x i8> @llvm.masked.gather.v4i8.v4p0(<4 x ptr> align 1 %ptrs, <4 x i1> %cond, <4 x i8> undef)224; CHECK-SVE-256-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret <4 x i8> %lv225;226; CHECK-SVE-512-LABEL: 'gather_load_4xi8_variable_mask'227; CHECK-SVE-512-NEXT: Cost Model: Found costs of 40 for: %lv = call <4 x i8> @llvm.masked.gather.v4i8.v4p0(<4 x ptr> align 1 %ptrs, <4 x i1> %cond, <4 x i8> undef)228; CHECK-SVE-512-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret <4 x i8> %lv229;230 %lv = call <4 x i8> @llvm.masked.gather.v4i8.v4p0(<4 x ptr> %ptrs, i32 1, <4 x i1> %cond, <4 x i8> undef)231 ret <4 x i8> %lv232}233 234declare void @llvm.masked.scatter.v4i8.v4p0(<4 x i8>, <4 x ptr>, i32 immarg, <4 x i1>)235define void @scatter_store_4xi8_constant_mask(<4 x i8> %val, <4 x ptr> %ptrs) {236; CHECK: scatter_store_4xi8_constant_mask237; CHECK-NEON-LABEL: 'scatter_store_4xi8_constant_mask'238; CHECK-NEON-NEXT: Cost Model: Found costs of RThru:20 CodeSize:12 Lat:20 SizeLat:20 for: call void @llvm.masked.scatter.v4i8.v4p0(<4 x i8> %val, <4 x ptr> align 1 %ptrs, <4 x i1> splat (i1 true))239; CHECK-NEON-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void240;241; CHECK-SVE-128-LABEL: 'scatter_store_4xi8_constant_mask'242; CHECK-SVE-128-NEXT: Cost Model: Found costs of RThru:20 CodeSize:12 Lat:20 SizeLat:20 for: call void @llvm.masked.scatter.v4i8.v4p0(<4 x i8> %val, <4 x ptr> align 1 %ptrs, <4 x i1> splat (i1 true))243; CHECK-SVE-128-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void244;245; CHECK-SVE-256-LABEL: 'scatter_store_4xi8_constant_mask'246; CHECK-SVE-256-NEXT: Cost Model: Found costs of 40 for: call void @llvm.masked.scatter.v4i8.v4p0(<4 x i8> %val, <4 x ptr> align 1 %ptrs, <4 x i1> splat (i1 true))247; CHECK-SVE-256-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void248;249; CHECK-SVE-512-LABEL: 'scatter_store_4xi8_constant_mask'250; CHECK-SVE-512-NEXT: Cost Model: Found costs of 40 for: call void @llvm.masked.scatter.v4i8.v4p0(<4 x i8> %val, <4 x ptr> align 1 %ptrs, <4 x i1> splat (i1 true))251; CHECK-SVE-512-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void252;253 call void @llvm.masked.scatter.v4i8.v4p0(<4 x i8> %val, <4 x ptr> %ptrs, i32 1, <4 x i1> <i1 true, i1 true, i1 true, i1 true>)254 ret void255}256 257define void @scatter_store_4xi8_variable_mask(<4 x i8> %val, <4 x ptr> %ptrs, <4 x i1> %cond) {258; CHECK: scatter_store_4xi8_variable_mask259; CHECK-NEON-LABEL: 'scatter_store_4xi8_variable_mask'260; CHECK-NEON-NEXT: Cost Model: Found costs of RThru:28 CodeSize:20 Lat:32 SizeLat:32 for: call void @llvm.masked.scatter.v4i8.v4p0(<4 x i8> %val, <4 x ptr> align 1 %ptrs, <4 x i1> %cond)261; CHECK-NEON-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void262;263; CHECK-SVE-128-LABEL: 'scatter_store_4xi8_variable_mask'264; CHECK-SVE-128-NEXT: Cost Model: Found costs of RThru:28 CodeSize:20 Lat:32 SizeLat:32 for: call void @llvm.masked.scatter.v4i8.v4p0(<4 x i8> %val, <4 x ptr> align 1 %ptrs, <4 x i1> %cond)265; CHECK-SVE-128-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void266;267; CHECK-SVE-256-LABEL: 'scatter_store_4xi8_variable_mask'268; CHECK-SVE-256-NEXT: Cost Model: Found costs of 40 for: call void @llvm.masked.scatter.v4i8.v4p0(<4 x i8> %val, <4 x ptr> align 1 %ptrs, <4 x i1> %cond)269; CHECK-SVE-256-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void270;271; CHECK-SVE-512-LABEL: 'scatter_store_4xi8_variable_mask'272; CHECK-SVE-512-NEXT: Cost Model: Found costs of 40 for: call void @llvm.masked.scatter.v4i8.v4p0(<4 x i8> %val, <4 x ptr> align 1 %ptrs, <4 x i1> %cond)273; CHECK-SVE-512-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void274;275 call void @llvm.masked.scatter.v4i8.v4p0(<4 x i8> %val, <4 x ptr> %ptrs, i32 1, <4 x i1> %cond)276 ret void277}278 279declare <4 x i32> @llvm.masked.gather.v4i32.v4p0(<4 x ptr>, i32 immarg, <4 x i1>, <4 x i32>)280define <4 x i32> @gather_load_4xi32_constant_mask(<4 x ptr> %ptrs) {281; CHECK: gather_load_4xi32_constant_mask282; CHECK-NEON-LABEL: 'gather_load_4xi32_constant_mask'283; CHECK-NEON-NEXT: Cost Model: Found costs of RThru:20 CodeSize:12 Lat:20 SizeLat:20 for: %lv = call <4 x i32> @llvm.masked.gather.v4i32.v4p0(<4 x ptr> align 1 %ptrs, <4 x i1> splat (i1 true), <4 x i32> undef)284; CHECK-NEON-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret <4 x i32> %lv285;286; CHECK-SVE-128-LABEL: 'gather_load_4xi32_constant_mask'287; CHECK-SVE-128-NEXT: Cost Model: Found costs of RThru:20 CodeSize:12 Lat:20 SizeLat:20 for: %lv = call <4 x i32> @llvm.masked.gather.v4i32.v4p0(<4 x ptr> align 1 %ptrs, <4 x i1> splat (i1 true), <4 x i32> undef)288; CHECK-SVE-128-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret <4 x i32> %lv289;290; CHECK-SVE-256-LABEL: 'gather_load_4xi32_constant_mask'291; CHECK-SVE-256-NEXT: Cost Model: Found costs of 40 for: %lv = call <4 x i32> @llvm.masked.gather.v4i32.v4p0(<4 x ptr> align 1 %ptrs, <4 x i1> splat (i1 true), <4 x i32> undef)292; CHECK-SVE-256-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret <4 x i32> %lv293;294; CHECK-SVE-512-LABEL: 'gather_load_4xi32_constant_mask'295; CHECK-SVE-512-NEXT: Cost Model: Found costs of 40 for: %lv = call <4 x i32> @llvm.masked.gather.v4i32.v4p0(<4 x ptr> align 1 %ptrs, <4 x i1> splat (i1 true), <4 x i32> undef)296; CHECK-SVE-512-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret <4 x i32> %lv297;298 %lv = call <4 x i32> @llvm.masked.gather.v4i32.v4p0(<4 x ptr> %ptrs, i32 1, <4 x i1> <i1 true, i1 true, i1 true, i1 true>, <4 x i32> undef)299 ret <4 x i32> %lv300}301 302define <4 x i32> @gather_load_4xi32_variable_mask(<4 x ptr> %ptrs, <4 x i1> %cond) {303; CHECK: gather_load_4xi32_variable_mask304; CHECK-NEON-LABEL: 'gather_load_4xi32_variable_mask'305; CHECK-NEON-NEXT: Cost Model: Found costs of RThru:28 CodeSize:20 Lat:32 SizeLat:32 for: %lv = call <4 x i32> @llvm.masked.gather.v4i32.v4p0(<4 x ptr> align 1 %ptrs, <4 x i1> %cond, <4 x i32> undef)306; CHECK-NEON-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret <4 x i32> %lv307;308; CHECK-SVE-128-LABEL: 'gather_load_4xi32_variable_mask'309; CHECK-SVE-128-NEXT: Cost Model: Found costs of RThru:28 CodeSize:20 Lat:32 SizeLat:32 for: %lv = call <4 x i32> @llvm.masked.gather.v4i32.v4p0(<4 x ptr> align 1 %ptrs, <4 x i1> %cond, <4 x i32> undef)310; CHECK-SVE-128-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret <4 x i32> %lv311;312; CHECK-SVE-256-LABEL: 'gather_load_4xi32_variable_mask'313; CHECK-SVE-256-NEXT: Cost Model: Found costs of 40 for: %lv = call <4 x i32> @llvm.masked.gather.v4i32.v4p0(<4 x ptr> align 1 %ptrs, <4 x i1> %cond, <4 x i32> undef)314; CHECK-SVE-256-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret <4 x i32> %lv315;316; CHECK-SVE-512-LABEL: 'gather_load_4xi32_variable_mask'317; CHECK-SVE-512-NEXT: Cost Model: Found costs of 40 for: %lv = call <4 x i32> @llvm.masked.gather.v4i32.v4p0(<4 x ptr> align 1 %ptrs, <4 x i1> %cond, <4 x i32> undef)318; CHECK-SVE-512-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret <4 x i32> %lv319;320 %lv = call <4 x i32> @llvm.masked.gather.v4i32.v4p0(<4 x ptr> %ptrs, i32 1, <4 x i1> %cond, <4 x i32> undef)321 ret <4 x i32> %lv322}323 324declare void @llvm.masked.scatter.v4i32.v4p0(<4 x i32>, <4 x ptr>, i32 immarg, <4 x i1>)325define void @scatter_store_4xi32_constant_mask(<4 x i32> %val, <4 x ptr> %ptrs) {326; CHECK: scatter_store_4xi32_constant_mask327; CHECK-NEON-LABEL: 'scatter_store_4xi32_constant_mask'328; CHECK-NEON-NEXT: Cost Model: Found costs of RThru:20 CodeSize:12 Lat:20 SizeLat:20 for: call void @llvm.masked.scatter.v4i32.v4p0(<4 x i32> %val, <4 x ptr> align 1 %ptrs, <4 x i1> splat (i1 true))329; CHECK-NEON-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void330;331; CHECK-SVE-128-LABEL: 'scatter_store_4xi32_constant_mask'332; CHECK-SVE-128-NEXT: Cost Model: Found costs of RThru:20 CodeSize:12 Lat:20 SizeLat:20 for: call void @llvm.masked.scatter.v4i32.v4p0(<4 x i32> %val, <4 x ptr> align 1 %ptrs, <4 x i1> splat (i1 true))333; CHECK-SVE-128-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void334;335; CHECK-SVE-256-LABEL: 'scatter_store_4xi32_constant_mask'336; CHECK-SVE-256-NEXT: Cost Model: Found costs of 40 for: call void @llvm.masked.scatter.v4i32.v4p0(<4 x i32> %val, <4 x ptr> align 1 %ptrs, <4 x i1> splat (i1 true))337; CHECK-SVE-256-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void338;339; CHECK-SVE-512-LABEL: 'scatter_store_4xi32_constant_mask'340; CHECK-SVE-512-NEXT: Cost Model: Found costs of 40 for: call void @llvm.masked.scatter.v4i32.v4p0(<4 x i32> %val, <4 x ptr> align 1 %ptrs, <4 x i1> splat (i1 true))341; CHECK-SVE-512-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void342;343 call void @llvm.masked.scatter.v4i32.v4p0(<4 x i32> %val, <4 x ptr> %ptrs, i32 1, <4 x i1> <i1 true, i1 true, i1 true, i1 true>)344 ret void345}346 347define void @scatter_store_4xi32_variable_mask(<4 x i32> %val, <4 x ptr> %ptrs, <4 x i1> %cond) {348; CHECK: scatter_store_4xi32_variable_mask349; CHECK-NEON-LABEL: 'scatter_store_4xi32_variable_mask'350; CHECK-NEON-NEXT: Cost Model: Found costs of RThru:28 CodeSize:20 Lat:32 SizeLat:32 for: call void @llvm.masked.scatter.v4i32.v4p0(<4 x i32> %val, <4 x ptr> align 1 %ptrs, <4 x i1> %cond)351; CHECK-NEON-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void352;353; CHECK-SVE-128-LABEL: 'scatter_store_4xi32_variable_mask'354; CHECK-SVE-128-NEXT: Cost Model: Found costs of RThru:28 CodeSize:20 Lat:32 SizeLat:32 for: call void @llvm.masked.scatter.v4i32.v4p0(<4 x i32> %val, <4 x ptr> align 1 %ptrs, <4 x i1> %cond)355; CHECK-SVE-128-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void356;357; CHECK-SVE-256-LABEL: 'scatter_store_4xi32_variable_mask'358; CHECK-SVE-256-NEXT: Cost Model: Found costs of 40 for: call void @llvm.masked.scatter.v4i32.v4p0(<4 x i32> %val, <4 x ptr> align 1 %ptrs, <4 x i1> %cond)359; CHECK-SVE-256-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void360;361; CHECK-SVE-512-LABEL: 'scatter_store_4xi32_variable_mask'362; CHECK-SVE-512-NEXT: Cost Model: Found costs of 40 for: call void @llvm.masked.scatter.v4i32.v4p0(<4 x i32> %val, <4 x ptr> align 1 %ptrs, <4 x i1> %cond)363; CHECK-SVE-512-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void364;365 call void @llvm.masked.scatter.v4i32.v4p0(<4 x i32> %val, <4 x ptr> %ptrs, i32 1, <4 x i1> %cond)366 ret void367}368 369declare <256 x i16> @llvm.masked.gather.v256i16.v256p0(<256 x ptr>, i32, <256 x i1>, <256 x i16>)370define void @sve_gather_vls(<256 x i1> %v256i1mask) {371; CHECK-LABEL: 'sve_scatter_vls'372; CHECK-NEON-LABEL: 'sve_gather_vls'373; CHECK-NEON-NEXT: Cost Model: Found costs of RThru:1792 CodeSize:1280 Lat:2048 SizeLat:2048 for: %res.v256i16 = call <256 x i16> @llvm.masked.gather.v256i16.v256p0(<256 x ptr> align 2 undef, <256 x i1> %v256i1mask, <256 x i16> zeroinitializer)374; CHECK-NEON-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void375;376; CHECK-SVE-128-LABEL: 'sve_gather_vls'377; CHECK-SVE-128-NEXT: Cost Model: Found costs of RThru:1792 CodeSize:1280 Lat:2048 SizeLat:2048 for: %res.v256i16 = call <256 x i16> @llvm.masked.gather.v256i16.v256p0(<256 x ptr> align 2 undef, <256 x i1> %v256i1mask, <256 x i16> zeroinitializer)378; CHECK-SVE-128-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void379;380; CHECK-SVE-256-LABEL: 'sve_gather_vls'381; CHECK-SVE-256-NEXT: Cost Model: Found costs of 2560 for: %res.v256i16 = call <256 x i16> @llvm.masked.gather.v256i16.v256p0(<256 x ptr> align 2 undef, <256 x i1> %v256i1mask, <256 x i16> zeroinitializer)382; CHECK-SVE-256-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void383;384; CHECK-SVE-512-LABEL: 'sve_gather_vls'385; CHECK-SVE-512-NEXT: Cost Model: Found costs of 2560 for: %res.v256i16 = call <256 x i16> @llvm.masked.gather.v256i16.v256p0(<256 x ptr> align 2 undef, <256 x i1> %v256i1mask, <256 x i16> zeroinitializer)386; CHECK-SVE-512-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void387;388entry:389 %res.v256i16 = call <256 x i16> @llvm.masked.gather.v256i16.v256p0(<256 x ptr> undef, i32 0, <256 x i1> %v256i1mask, <256 x i16> zeroinitializer)390 ret void391}392 393declare <256 x float> @llvm.masked.gather.v256f32.v256p0(<256 x ptr>, i32, <256 x i1>, <256 x float>)394define void @sve_gather_vls_float(<256 x i1> %v256i1mask) {395; CHECK-LABEL: 'sve_gather_vls_float'396; CHECK-NEON-LABEL: 'sve_gather_vls_float'397; CHECK-NEON-NEXT: Cost Model: Found costs of RThru:1664 CodeSize:1216 Lat:1920 SizeLat:1920 for: %res.v256f32 = call <256 x float> @llvm.masked.gather.v256f32.v256p0(<256 x ptr> align 4 undef, <256 x i1> %v256i1mask, <256 x float> zeroinitializer)398; CHECK-NEON-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void399;400; CHECK-SVE-128-LABEL: 'sve_gather_vls_float'401; CHECK-SVE-128-NEXT: Cost Model: Found costs of RThru:1664 CodeSize:1216 Lat:1920 SizeLat:1920 for: %res.v256f32 = call <256 x float> @llvm.masked.gather.v256f32.v256p0(<256 x ptr> align 4 undef, <256 x i1> %v256i1mask, <256 x float> zeroinitializer)402; CHECK-SVE-128-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void403;404; CHECK-SVE-256-LABEL: 'sve_gather_vls_float'405; CHECK-SVE-256-NEXT: Cost Model: Found costs of 2560 for: %res.v256f32 = call <256 x float> @llvm.masked.gather.v256f32.v256p0(<256 x ptr> align 4 undef, <256 x i1> %v256i1mask, <256 x float> zeroinitializer)406; CHECK-SVE-256-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void407;408; CHECK-SVE-512-LABEL: 'sve_gather_vls_float'409; CHECK-SVE-512-NEXT: Cost Model: Found costs of 2560 for: %res.v256f32 = call <256 x float> @llvm.masked.gather.v256f32.v256p0(<256 x ptr> align 4 undef, <256 x i1> %v256i1mask, <256 x float> zeroinitializer)410; CHECK-SVE-512-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void411;412entry:413 %res.v256f32 = call <256 x float> @llvm.masked.gather.v256f32.v256p0(<256 x ptr> undef, i32 0, <256 x i1> %v256i1mask, <256 x float> zeroinitializer)414 ret void415}416 417declare void @llvm.masked.scatter.v256i8.v256p0(<256 x i8>, <256 x ptr>, i32, <256 x i1>)418define void @sve_scatter_vls(<256 x i1> %v256i1mask){419; CHECK-LABEL: 'sve_scatter_vls'420; CHECK-NEON-LABEL: 'sve_scatter_vls'421; CHECK-NEON-NEXT: Cost Model: Found costs of RThru:1792 CodeSize:1280 Lat:2048 SizeLat:2048 for: call void @llvm.masked.scatter.v256i8.v256p0(<256 x i8> undef, <256 x ptr> align 1 undef, <256 x i1> %v256i1mask)422; CHECK-NEON-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void423;424; CHECK-SVE-128-LABEL: 'sve_scatter_vls'425; CHECK-SVE-128-NEXT: Cost Model: Found costs of RThru:1792 CodeSize:1280 Lat:2048 SizeLat:2048 for: call void @llvm.masked.scatter.v256i8.v256p0(<256 x i8> undef, <256 x ptr> align 1 undef, <256 x i1> %v256i1mask)426; CHECK-SVE-128-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void427;428; CHECK-SVE-256-LABEL: 'sve_scatter_vls'429; CHECK-SVE-256-NEXT: Cost Model: Found costs of 2560 for: call void @llvm.masked.scatter.v256i8.v256p0(<256 x i8> undef, <256 x ptr> align 1 undef, <256 x i1> %v256i1mask)430; CHECK-SVE-256-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void431;432; CHECK-SVE-512-LABEL: 'sve_scatter_vls'433; CHECK-SVE-512-NEXT: Cost Model: Found costs of 2560 for: call void @llvm.masked.scatter.v256i8.v256p0(<256 x i8> undef, <256 x ptr> align 1 undef, <256 x i1> %v256i1mask)434; CHECK-SVE-512-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void435;436entry:437 call void @llvm.masked.scatter.v256i8.v256p0(<256 x i8> undef, <256 x ptr> undef, i32 0, <256 x i1> %v256i1mask)438 ret void439}440 441declare void @llvm.masked.scatter.v512f16.v512p0(<512 x half>, <512 x ptr>, i32, <512 x i1>)442define void @sve_scatter_vls_float(<512 x i1> %v512i1mask){443; CHECK-LABEL: 'sve_scatter_vls_float'444; CHECK-NEON-LABEL: 'sve_scatter_vls_float'445; CHECK-NEON-NEXT: Cost Model: Found costs of RThru:3456 CodeSize:2496 Lat:3968 SizeLat:3968 for: call void @llvm.masked.scatter.v512f16.v512p0(<512 x half> undef, <512 x ptr> align 2 undef, <512 x i1> %v512i1mask)446; CHECK-NEON-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void447;448; CHECK-SVE-128-LABEL: 'sve_scatter_vls_float'449; CHECK-SVE-128-NEXT: Cost Model: Found costs of RThru:3456 CodeSize:2496 Lat:3968 SizeLat:3968 for: call void @llvm.masked.scatter.v512f16.v512p0(<512 x half> undef, <512 x ptr> align 2 undef, <512 x i1> %v512i1mask)450; CHECK-SVE-128-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void451;452; CHECK-SVE-256-LABEL: 'sve_scatter_vls_float'453; CHECK-SVE-256-NEXT: Cost Model: Found costs of 5120 for: call void @llvm.masked.scatter.v512f16.v512p0(<512 x half> undef, <512 x ptr> align 2 undef, <512 x i1> %v512i1mask)454; CHECK-SVE-256-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void455;456; CHECK-SVE-512-LABEL: 'sve_scatter_vls_float'457; CHECK-SVE-512-NEXT: Cost Model: Found costs of 5120 for: call void @llvm.masked.scatter.v512f16.v512p0(<512 x half> undef, <512 x ptr> align 2 undef, <512 x i1> %v512i1mask)458; CHECK-SVE-512-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void459;460 call void @llvm.masked.scatter.v512f16.v512p0(<512 x half> undef, <512 x ptr> undef, i32 0, <512 x i1> %v512i1mask)461 ret void462}463