198 lines · plain
1; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py2; RUN: llc -mtriple=aarch64--linux-gnu -mattr=+sve -aarch64-enable-mgather-combine=0 < %s | FileCheck %s3; RUN: llc -mtriple=aarch64--linux-gnu -mattr=+sve -aarch64-enable-mgather-combine=1 < %s | FileCheck %s4 5;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;6; unscaled unpacked 32-bit offsets7;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;8 9define <vscale x 2 x i64> @masked_gather_nxv2i16(ptr %base, <vscale x 2 x i32> %offsets, <vscale x 2 x i1> %mask) {10; CHECK-LABEL: masked_gather_nxv2i16:11; CHECK: // %bb.0:12; CHECK-NEXT: ld1h { z0.d }, p0/z, [x0, z0.d, uxtw #1]13; CHECK-NEXT: ret14 %offsets.zext = zext <vscale x 2 x i32> %offsets to <vscale x 2 x i64>15 %ptrs = getelementptr i16, ptr %base, <vscale x 2 x i64> %offsets.zext16 %vals = call <vscale x 2 x i16> @llvm.masked.gather.nxv2i16(<vscale x 2 x ptr> %ptrs, i32 2, <vscale x 2 x i1> %mask, <vscale x 2 x i16> poison)17 %vals.zext = zext <vscale x 2 x i16> %vals to <vscale x 2 x i64>18 ret <vscale x 2 x i64> %vals.zext19}20 21define <vscale x 2 x i64> @masked_gather_nxv2i32(ptr %base, <vscale x 2 x i32> %offsets, <vscale x 2 x i1> %mask) {22; CHECK-LABEL: masked_gather_nxv2i32:23; CHECK: // %bb.0:24; CHECK-NEXT: ld1w { z0.d }, p0/z, [x0, z0.d, uxtw #2]25; CHECK-NEXT: ret26 %offsets.zext = zext <vscale x 2 x i32> %offsets to <vscale x 2 x i64>27 %ptrs = getelementptr i32, ptr %base, <vscale x 2 x i64> %offsets.zext28 %vals = call <vscale x 2 x i32> @llvm.masked.gather.nxv2i32(<vscale x 2 x ptr> %ptrs, i32 4, <vscale x 2 x i1> %mask, <vscale x 2 x i32> poison)29 %vals.zext = zext <vscale x 2 x i32> %vals to <vscale x 2 x i64>30 ret <vscale x 2 x i64> %vals.zext31}32 33define <vscale x 2 x i64> @masked_gather_nxv2i64(ptr %base, <vscale x 2 x i32> %offsets, <vscale x 2 x i1> %mask) {34; CHECK-LABEL: masked_gather_nxv2i64:35; CHECK: // %bb.0:36; CHECK-NEXT: ld1d { z0.d }, p0/z, [x0, z0.d, uxtw #3]37; CHECK-NEXT: ret38 %offsets.zext = zext <vscale x 2 x i32> %offsets to <vscale x 2 x i64>39 %ptrs = getelementptr i64, ptr %base, <vscale x 2 x i64> %offsets.zext40 %vals = call <vscale x 2 x i64> @llvm.masked.gather.nxv2i64(<vscale x 2 x ptr> %ptrs, i32 8, <vscale x 2 x i1> %mask, <vscale x 2 x i64> poison)41 ret <vscale x 2 x i64> %vals42}43 44define <vscale x 2 x half> @masked_gather_nxv2f16(ptr %base, <vscale x 2 x i32> %offsets, <vscale x 2 x i1> %mask) {45; CHECK-LABEL: masked_gather_nxv2f16:46; CHECK: // %bb.0:47; CHECK-NEXT: ld1h { z0.d }, p0/z, [x0, z0.d, uxtw #1]48; CHECK-NEXT: ret49 %offsets.zext = zext <vscale x 2 x i32> %offsets to <vscale x 2 x i64>50 %ptrs = getelementptr half, ptr %base, <vscale x 2 x i64> %offsets.zext51 %vals = call <vscale x 2 x half> @llvm.masked.gather.nxv2f16(<vscale x 2 x ptr> %ptrs, i32 2, <vscale x 2 x i1> %mask, <vscale x 2 x half> poison)52 ret <vscale x 2 x half> %vals53}54 55define <vscale x 2 x bfloat> @masked_gather_nxv2bf16(ptr %base, <vscale x 2 x i32> %offsets, <vscale x 2 x i1> %mask) #0 {56; CHECK-LABEL: masked_gather_nxv2bf16:57; CHECK: // %bb.0:58; CHECK-NEXT: ld1h { z0.d }, p0/z, [x0, z0.d, uxtw #1]59; CHECK-NEXT: ret60 %offsets.zext = zext <vscale x 2 x i32> %offsets to <vscale x 2 x i64>61 %ptrs = getelementptr bfloat, ptr %base, <vscale x 2 x i64> %offsets.zext62 %vals = call <vscale x 2 x bfloat> @llvm.masked.gather.nxv2bf16(<vscale x 2 x ptr> %ptrs, i32 2, <vscale x 2 x i1> %mask, <vscale x 2 x bfloat> poison)63 ret <vscale x 2 x bfloat> %vals64}65 66define <vscale x 2 x float> @masked_gather_nxv2f32(ptr %base, <vscale x 2 x i32> %offsets, <vscale x 2 x i1> %mask) {67; CHECK-LABEL: masked_gather_nxv2f32:68; CHECK: // %bb.0:69; CHECK-NEXT: ld1w { z0.d }, p0/z, [x0, z0.d, uxtw #2]70; CHECK-NEXT: ret71 %offsets.zext = zext <vscale x 2 x i32> %offsets to <vscale x 2 x i64>72 %ptrs = getelementptr float, ptr %base, <vscale x 2 x i64> %offsets.zext73 %vals = call <vscale x 2 x float> @llvm.masked.gather.nxv2f32(<vscale x 2 x ptr> %ptrs, i32 4, <vscale x 2 x i1> %mask, <vscale x 2 x float> poison)74 ret <vscale x 2 x float> %vals75}76 77define <vscale x 2 x double> @masked_gather_nxv2f64(ptr %base, <vscale x 2 x i32> %offsets, <vscale x 2 x i1> %mask) {78; CHECK-LABEL: masked_gather_nxv2f64:79; CHECK: // %bb.0:80; CHECK-NEXT: ld1d { z0.d }, p0/z, [x0, z0.d, uxtw #3]81; CHECK-NEXT: ret82 %offsets.zext = zext <vscale x 2 x i32> %offsets to <vscale x 2 x i64>83 %ptrs = getelementptr double, ptr %base, <vscale x 2 x i64> %offsets.zext84 %vals = call <vscale x 2 x double> @llvm.masked.gather.nxv2f64(<vscale x 2 x ptr> %ptrs, i32 8, <vscale x 2 x i1> %mask, <vscale x 2 x double> poison)85 ret <vscale x 2 x double> %vals86}87 88define <vscale x 2 x i64> @masked_sgather_nxv2i16(ptr %base, <vscale x 2 x i32> %offsets, <vscale x 2 x i1> %mask) {89; CHECK-LABEL: masked_sgather_nxv2i16:90; CHECK: // %bb.0:91; CHECK-NEXT: ld1sh { z0.d }, p0/z, [x0, z0.d, uxtw #1]92; CHECK-NEXT: ret93 %offsets.zext = zext <vscale x 2 x i32> %offsets to <vscale x 2 x i64>94 %ptrs = getelementptr i16, ptr %base, <vscale x 2 x i64> %offsets.zext95 %vals = call <vscale x 2 x i16> @llvm.masked.gather.nxv2i16(<vscale x 2 x ptr> %ptrs, i32 2, <vscale x 2 x i1> %mask, <vscale x 2 x i16> poison)96 %vals.sext = sext <vscale x 2 x i16> %vals to <vscale x 2 x i64>97 ret <vscale x 2 x i64> %vals.sext98}99 100define <vscale x 2 x i64> @masked_sgather_nxv2i32(ptr %base, <vscale x 2 x i32> %offsets, <vscale x 2 x i1> %mask) {101; CHECK-LABEL: masked_sgather_nxv2i32:102; CHECK: // %bb.0:103; CHECK-NEXT: ld1sw { z0.d }, p0/z, [x0, z0.d, uxtw #2]104; CHECK-NEXT: ret105 %offsets.zext = zext <vscale x 2 x i32> %offsets to <vscale x 2 x i64>106 %ptrs = getelementptr i32, ptr %base, <vscale x 2 x i64> %offsets.zext107 %vals = call <vscale x 2 x i32> @llvm.masked.gather.nxv2i32(<vscale x 2 x ptr> %ptrs, i32 4, <vscale x 2 x i1> %mask, <vscale x 2 x i32> poison)108 %vals.sext = sext <vscale x 2 x i32> %vals to <vscale x 2 x i64>109 ret <vscale x 2 x i64> %vals.sext110}111 112;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;113; unscaled packed 32-bit offsets114;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;115 116define <vscale x 4 x i32> @masked_gather_nxv4i16(ptr %base, <vscale x 4 x i32> %offsets, <vscale x 4 x i1> %mask) {117; CHECK-LABEL: masked_gather_nxv4i16:118; CHECK: // %bb.0:119; CHECK-NEXT: ld1h { z0.s }, p0/z, [x0, z0.s, uxtw #1]120; CHECK-NEXT: ret121 %offsets.zext = zext <vscale x 4 x i32> %offsets to <vscale x 4 x i64>122 %ptrs = getelementptr i16, ptr %base, <vscale x 4 x i64> %offsets.zext123 %vals = call <vscale x 4 x i16> @llvm.masked.gather.nxv4i16(<vscale x 4 x ptr> %ptrs, i32 2, <vscale x 4 x i1> %mask, <vscale x 4 x i16> poison)124 %vals.zext = zext <vscale x 4 x i16> %vals to <vscale x 4 x i32>125 ret <vscale x 4 x i32> %vals.zext126}127 128define <vscale x 4 x i32> @masked_gather_nxv4i32(ptr %base, <vscale x 4 x i32> %offsets, <vscale x 4 x i1> %mask) {129; CHECK-LABEL: masked_gather_nxv4i32:130; CHECK: // %bb.0:131; CHECK-NEXT: ld1w { z0.s }, p0/z, [x0, z0.s, uxtw #2]132; CHECK-NEXT: ret133 %offsets.zext = zext <vscale x 4 x i32> %offsets to <vscale x 4 x i64>134 %ptrs = getelementptr i32, ptr %base, <vscale x 4 x i64> %offsets.zext135 %vals = call <vscale x 4 x i32> @llvm.masked.gather.nxv4i32(<vscale x 4 x ptr> %ptrs, i32 4, <vscale x 4 x i1> %mask, <vscale x 4 x i32> poison)136 ret <vscale x 4 x i32> %vals137}138 139define <vscale x 4 x half> @masked_gather_nxv4f16(ptr %base, <vscale x 4 x i32> %offsets, <vscale x 4 x i1> %mask) {140; CHECK-LABEL: masked_gather_nxv4f16:141; CHECK: // %bb.0:142; CHECK-NEXT: ld1h { z0.s }, p0/z, [x0, z0.s, uxtw #1]143; CHECK-NEXT: ret144 %offsets.zext = zext <vscale x 4 x i32> %offsets to <vscale x 4 x i64>145 %ptrs = getelementptr half, ptr %base, <vscale x 4 x i64> %offsets.zext146 %vals = call <vscale x 4 x half> @llvm.masked.gather.nxv4f16(<vscale x 4 x ptr> %ptrs, i32 2, <vscale x 4 x i1> %mask, <vscale x 4 x half> poison)147 ret <vscale x 4 x half> %vals148}149 150define <vscale x 4 x bfloat> @masked_gather_nxv4bf16(ptr %base, <vscale x 4 x i32> %offsets, <vscale x 4 x i1> %mask) #0 {151; CHECK-LABEL: masked_gather_nxv4bf16:152; CHECK: // %bb.0:153; CHECK-NEXT: ld1h { z0.s }, p0/z, [x0, z0.s, uxtw #1]154; CHECK-NEXT: ret155 %offsets.zext = zext <vscale x 4 x i32> %offsets to <vscale x 4 x i64>156 %ptrs = getelementptr bfloat, ptr %base, <vscale x 4 x i64> %offsets.zext157 %vals = call <vscale x 4 x bfloat> @llvm.masked.gather.nxv4bf16(<vscale x 4 x ptr> %ptrs, i32 2, <vscale x 4 x i1> %mask, <vscale x 4 x bfloat> poison)158 ret <vscale x 4 x bfloat> %vals159}160 161define <vscale x 4 x float> @masked_gather_nxv4f32(ptr %base, <vscale x 4 x i32> %offsets, <vscale x 4 x i1> %mask) {162; CHECK-LABEL: masked_gather_nxv4f32:163; CHECK: // %bb.0:164; CHECK-NEXT: ld1w { z0.s }, p0/z, [x0, z0.s, uxtw #2]165; CHECK-NEXT: ret166 %offsets.zext = zext <vscale x 4 x i32> %offsets to <vscale x 4 x i64>167 %ptrs = getelementptr float, ptr %base, <vscale x 4 x i64> %offsets.zext168 %vals = call <vscale x 4 x float> @llvm.masked.gather.nxv4f32(<vscale x 4 x ptr> %ptrs, i32 4, <vscale x 4 x i1> %mask, <vscale x 4 x float> poison)169 ret <vscale x 4 x float> %vals170}171 172define <vscale x 4 x i32> @masked_sgather_nxv4i16(ptr %base, <vscale x 4 x i32> %offsets, <vscale x 4 x i1> %mask) {173; CHECK-LABEL: masked_sgather_nxv4i16:174; CHECK: // %bb.0:175; CHECK-NEXT: ld1sh { z0.s }, p0/z, [x0, z0.s, uxtw #1]176; CHECK-NEXT: ret177 %offsets.zext = zext <vscale x 4 x i32> %offsets to <vscale x 4 x i64>178 %ptrs = getelementptr i16, ptr %base, <vscale x 4 x i64> %offsets.zext179 %vals = call <vscale x 4 x i16> @llvm.masked.gather.nxv4i16(<vscale x 4 x ptr> %ptrs, i32 2, <vscale x 4 x i1> %mask, <vscale x 4 x i16> poison)180 %vals.sext = sext <vscale x 4 x i16> %vals to <vscale x 4 x i32>181 ret <vscale x 4 x i32> %vals.sext182}183 184declare <vscale x 2 x i16> @llvm.masked.gather.nxv2i16(<vscale x 2 x ptr>, i32, <vscale x 2 x i1>, <vscale x 2 x i16>)185declare <vscale x 2 x i32> @llvm.masked.gather.nxv2i32(<vscale x 2 x ptr>, i32, <vscale x 2 x i1>, <vscale x 2 x i32>)186declare <vscale x 2 x i64> @llvm.masked.gather.nxv2i64(<vscale x 2 x ptr>, i32, <vscale x 2 x i1>, <vscale x 2 x i64>)187declare <vscale x 2 x half> @llvm.masked.gather.nxv2f16(<vscale x 2 x ptr>, i32, <vscale x 2 x i1>, <vscale x 2 x half>)188declare <vscale x 2 x bfloat> @llvm.masked.gather.nxv2bf16(<vscale x 2 x ptr>, i32, <vscale x 2 x i1>, <vscale x 2 x bfloat>)189declare <vscale x 2 x float> @llvm.masked.gather.nxv2f32(<vscale x 2 x ptr>, i32, <vscale x 2 x i1>, <vscale x 2 x float>)190declare <vscale x 2 x double> @llvm.masked.gather.nxv2f64(<vscale x 2 x ptr>, i32, <vscale x 2 x i1>, <vscale x 2 x double>)191 192declare <vscale x 4 x i16> @llvm.masked.gather.nxv4i16(<vscale x 4 x ptr>, i32, <vscale x 4 x i1>, <vscale x 4 x i16>)193declare <vscale x 4 x i32> @llvm.masked.gather.nxv4i32(<vscale x 4 x ptr>, i32, <vscale x 4 x i1>, <vscale x 4 x i32>)194declare <vscale x 4 x half> @llvm.masked.gather.nxv4f16(<vscale x 4 x ptr>, i32, <vscale x 4 x i1>, <vscale x 4 x half>)195declare <vscale x 4 x bfloat> @llvm.masked.gather.nxv4bf16(<vscale x 4 x ptr>, i32, <vscale x 4 x i1>, <vscale x 4 x bfloat>)196declare <vscale x 4 x float> @llvm.masked.gather.nxv4f32(<vscale x 4 x ptr>, i32, <vscale x 4 x i1>, <vscale x 4 x float>)197attributes #0 = { "target-features"="+sve,+bf16" }198