263 lines · plain
1; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py2; RUN: llc -mtriple=aarch64--linux-gnu -mattr=+sve -aarch64-enable-mgather-combine=0 < %s | FileCheck %s3; RUN: llc -mtriple=aarch64--linux-gnu -mattr=+sve -aarch64-enable-mgather-combine=1 < %s | FileCheck %s4 5;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;6; unscaled unpacked 32-bit offsets7;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;8 9define <vscale x 2 x i64> @masked_gather_nxv2i8(ptr %base, <vscale x 2 x i32> %offsets, <vscale x 2 x i1> %mask) {10; CHECK-LABEL: masked_gather_nxv2i8:11; CHECK: // %bb.0:12; CHECK-NEXT: ld1b { z0.d }, p0/z, [x0, z0.d, uxtw]13; CHECK-NEXT: ret14 %offsets.zext = zext <vscale x 2 x i32> %offsets to <vscale x 2 x i64>15 %ptrs = getelementptr i8, ptr %base, <vscale x 2 x i64> %offsets.zext16 %vals = call <vscale x 2 x i8> @llvm.masked.gather.nxv2i8(<vscale x 2 x ptr> %ptrs, i32 1, <vscale x 2 x i1> %mask, <vscale x 2 x i8> poison)17 %vals.zext = zext <vscale x 2 x i8> %vals to <vscale x 2 x i64>18 ret <vscale x 2 x i64> %vals.zext19}20 21define <vscale x 2 x i64> @masked_gather_nxv2i16(ptr %base, <vscale x 2 x i32> %offsets, <vscale x 2 x i1> %mask) {22; CHECK-LABEL: masked_gather_nxv2i16:23; CHECK: // %bb.0:24; CHECK-NEXT: ld1h { z0.d }, p0/z, [x0, z0.d, uxtw]25; CHECK-NEXT: ret26 %offsets.zext = zext <vscale x 2 x i32> %offsets to <vscale x 2 x i64>27 %byte_ptrs = getelementptr i8, ptr %base, <vscale x 2 x i64> %offsets.zext28 %ptrs = bitcast <vscale x 2 x ptr> %byte_ptrs to <vscale x 2 x ptr>29 %vals = call <vscale x 2 x i16> @llvm.masked.gather.nxv2i16(<vscale x 2 x ptr> %ptrs, i32 2, <vscale x 2 x i1> %mask, <vscale x 2 x i16> poison)30 %vals.zext = zext <vscale x 2 x i16> %vals to <vscale x 2 x i64>31 ret <vscale x 2 x i64> %vals.zext32}33 34define <vscale x 2 x i64> @masked_gather_nxv2i32(ptr %base, <vscale x 2 x i32> %offsets, <vscale x 2 x i1> %mask) {35; CHECK-LABEL: masked_gather_nxv2i32:36; CHECK: // %bb.0:37; CHECK-NEXT: ld1w { z0.d }, p0/z, [x0, z0.d, uxtw]38; CHECK-NEXT: ret39 %offsets.zext = zext <vscale x 2 x i32> %offsets to <vscale x 2 x i64>40 %byte_ptrs = getelementptr i8, ptr %base, <vscale x 2 x i64> %offsets.zext41 %ptrs = bitcast <vscale x 2 x ptr> %byte_ptrs to <vscale x 2 x ptr>42 %vals = call <vscale x 2 x i32> @llvm.masked.gather.nxv2i32(<vscale x 2 x ptr> %ptrs, i32 4, <vscale x 2 x i1> %mask, <vscale x 2 x i32> poison)43 %vals.zext = zext <vscale x 2 x i32> %vals to <vscale x 2 x i64>44 ret <vscale x 2 x i64> %vals.zext45}46 47define <vscale x 2 x i64> @masked_gather_nxv2i64(ptr %base, <vscale x 2 x i32> %offsets, <vscale x 2 x i1> %mask) {48; CHECK-LABEL: masked_gather_nxv2i64:49; CHECK: // %bb.0:50; CHECK-NEXT: ld1d { z0.d }, p0/z, [x0, z0.d, uxtw]51; CHECK-NEXT: ret52 %offsets.zext = zext <vscale x 2 x i32> %offsets to <vscale x 2 x i64>53 %byte_ptrs = getelementptr i8, ptr %base, <vscale x 2 x i64> %offsets.zext54 %ptrs = bitcast <vscale x 2 x ptr> %byte_ptrs to <vscale x 2 x ptr>55 %vals = call <vscale x 2 x i64> @llvm.masked.gather.nxv2i64(<vscale x 2 x ptr> %ptrs, i32 8, <vscale x 2 x i1> %mask, <vscale x 2 x i64> poison)56 ret <vscale x 2 x i64> %vals57}58 59define <vscale x 2 x half> @masked_gather_nxv2f16(ptr %base, <vscale x 2 x i32> %offsets, <vscale x 2 x i1> %mask) {60; CHECK-LABEL: masked_gather_nxv2f16:61; CHECK: // %bb.0:62; CHECK-NEXT: ld1h { z0.d }, p0/z, [x0, z0.d, uxtw]63; CHECK-NEXT: ret64 %offsets.zext = zext <vscale x 2 x i32> %offsets to <vscale x 2 x i64>65 %byte_ptrs = getelementptr i8, ptr %base, <vscale x 2 x i64> %offsets.zext66 %ptrs = bitcast <vscale x 2 x ptr> %byte_ptrs to <vscale x 2 x ptr>67 %vals = call <vscale x 2 x half> @llvm.masked.gather.nxv2f16(<vscale x 2 x ptr> %ptrs, i32 2, <vscale x 2 x i1> %mask, <vscale x 2 x half> poison)68 ret <vscale x 2 x half> %vals69}70 71define <vscale x 2 x bfloat> @masked_gather_nxv2bf16(ptr %base, <vscale x 2 x i32> %offsets, <vscale x 2 x i1> %mask) #0 {72; CHECK-LABEL: masked_gather_nxv2bf16:73; CHECK: // %bb.0:74; CHECK-NEXT: ld1h { z0.d }, p0/z, [x0, z0.d, uxtw]75; CHECK-NEXT: ret76 %offsets.zext = zext <vscale x 2 x i32> %offsets to <vscale x 2 x i64>77 %byte_ptrs = getelementptr i8, ptr %base, <vscale x 2 x i64> %offsets.zext78 %ptrs = bitcast <vscale x 2 x ptr> %byte_ptrs to <vscale x 2 x ptr>79 %vals = call <vscale x 2 x bfloat> @llvm.masked.gather.nxv2bf16(<vscale x 2 x ptr> %ptrs, i32 2, <vscale x 2 x i1> %mask, <vscale x 2 x bfloat> poison)80 ret <vscale x 2 x bfloat> %vals81}82 83define <vscale x 2 x float> @masked_gather_nxv2f32(ptr %base, <vscale x 2 x i32> %offsets, <vscale x 2 x i1> %mask) {84; CHECK-LABEL: masked_gather_nxv2f32:85; CHECK: // %bb.0:86; CHECK-NEXT: ld1w { z0.d }, p0/z, [x0, z0.d, uxtw]87; CHECK-NEXT: ret88 %offsets.zext = zext <vscale x 2 x i32> %offsets to <vscale x 2 x i64>89 %byte_ptrs = getelementptr i8, ptr %base, <vscale x 2 x i64> %offsets.zext90 %ptrs = bitcast <vscale x 2 x ptr> %byte_ptrs to <vscale x 2 x ptr>91 %vals = call <vscale x 2 x float> @llvm.masked.gather.nxv2f32(<vscale x 2 x ptr> %ptrs, i32 4, <vscale x 2 x i1> %mask, <vscale x 2 x float> poison)92 ret <vscale x 2 x float> %vals93}94 95define <vscale x 2 x double> @masked_gather_nxv2f64(ptr %base, <vscale x 2 x i32> %offsets, <vscale x 2 x i1> %mask) {96; CHECK-LABEL: masked_gather_nxv2f64:97; CHECK: // %bb.0:98; CHECK-NEXT: ld1d { z0.d }, p0/z, [x0, z0.d, uxtw]99; CHECK-NEXT: ret100 %offsets.zext = zext <vscale x 2 x i32> %offsets to <vscale x 2 x i64>101 %byte_ptrs = getelementptr i8, ptr %base, <vscale x 2 x i64> %offsets.zext102 %ptrs = bitcast <vscale x 2 x ptr> %byte_ptrs to <vscale x 2 x ptr>103 %vals = call <vscale x 2 x double> @llvm.masked.gather.nxv2f64(<vscale x 2 x ptr> %ptrs, i32 8, <vscale x 2 x i1> %mask, <vscale x 2 x double> poison)104 ret <vscale x 2 x double> %vals105}106 107define <vscale x 2 x i64> @masked_sgather_nxv2i8(ptr %base, <vscale x 2 x i32> %offsets, <vscale x 2 x i1> %mask) {108; CHECK-LABEL: masked_sgather_nxv2i8:109; CHECK: // %bb.0:110; CHECK-NEXT: ld1sb { z0.d }, p0/z, [x0, z0.d, uxtw]111; CHECK-NEXT: ret112 %offsets.zext = zext <vscale x 2 x i32> %offsets to <vscale x 2 x i64>113 %ptrs = getelementptr i8, ptr %base, <vscale x 2 x i64> %offsets.zext114 %vals = call <vscale x 2 x i8> @llvm.masked.gather.nxv2i8(<vscale x 2 x ptr> %ptrs, i32 1, <vscale x 2 x i1> %mask, <vscale x 2 x i8> poison)115 %vals.sext = sext <vscale x 2 x i8> %vals to <vscale x 2 x i64>116 ret <vscale x 2 x i64> %vals.sext117}118 119define <vscale x 2 x i64> @masked_sgather_nxv2i16(ptr %base, <vscale x 2 x i32> %offsets, <vscale x 2 x i1> %mask) {120; CHECK-LABEL: masked_sgather_nxv2i16:121; CHECK: // %bb.0:122; CHECK-NEXT: ld1sh { z0.d }, p0/z, [x0, z0.d, uxtw]123; CHECK-NEXT: ret124 %offsets.zext = zext <vscale x 2 x i32> %offsets to <vscale x 2 x i64>125 %byte_ptrs = getelementptr i8, ptr %base, <vscale x 2 x i64> %offsets.zext126 %ptrs = bitcast <vscale x 2 x ptr> %byte_ptrs to <vscale x 2 x ptr>127 %vals = call <vscale x 2 x i16> @llvm.masked.gather.nxv2i16(<vscale x 2 x ptr> %ptrs, i32 2, <vscale x 2 x i1> %mask, <vscale x 2 x i16> poison)128 %vals.sext = sext <vscale x 2 x i16> %vals to <vscale x 2 x i64>129 ret <vscale x 2 x i64> %vals.sext130}131 132define <vscale x 2 x i64> @masked_sgather_nxv2i32(ptr %base, <vscale x 2 x i32> %offsets, <vscale x 2 x i1> %mask) {133; CHECK-LABEL: masked_sgather_nxv2i32:134; CHECK: // %bb.0:135; CHECK-NEXT: ld1sw { z0.d }, p0/z, [x0, z0.d, uxtw]136; CHECK-NEXT: ret137 %offsets.zext = zext <vscale x 2 x i32> %offsets to <vscale x 2 x i64>138 %byte_ptrs = getelementptr i8, ptr %base, <vscale x 2 x i64> %offsets.zext139 %ptrs = bitcast <vscale x 2 x ptr> %byte_ptrs to <vscale x 2 x ptr>140 %vals = call <vscale x 2 x i32> @llvm.masked.gather.nxv2i32(<vscale x 2 x ptr> %ptrs, i32 4, <vscale x 2 x i1> %mask, <vscale x 2 x i32> poison)141 %vals.sext = sext <vscale x 2 x i32> %vals to <vscale x 2 x i64>142 ret <vscale x 2 x i64> %vals.sext143}144 145;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;146; unscaled packed 32-bit offsets147;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;148 149define <vscale x 4 x i32> @masked_gather_nxv4i8(ptr %base, <vscale x 4 x i32> %offsets, <vscale x 4 x i1> %mask) {150; CHECK-LABEL: masked_gather_nxv4i8:151; CHECK: // %bb.0:152; CHECK-NEXT: ld1b { z0.s }, p0/z, [x0, z0.s, uxtw]153; CHECK-NEXT: ret154 %offsets.zext = zext <vscale x 4 x i32> %offsets to <vscale x 4 x i64>155 %ptrs = getelementptr i8, ptr %base, <vscale x 4 x i64> %offsets.zext156 %vals = call <vscale x 4 x i8> @llvm.masked.gather.nxv4i8(<vscale x 4 x ptr> %ptrs, i32 1, <vscale x 4 x i1> %mask, <vscale x 4 x i8> poison)157 %vals.zext = zext <vscale x 4 x i8> %vals to <vscale x 4 x i32>158 ret <vscale x 4 x i32> %vals.zext159}160 161define <vscale x 4 x i32> @masked_gather_nxv4i16(ptr %base, <vscale x 4 x i32> %offsets, <vscale x 4 x i1> %mask) {162; CHECK-LABEL: masked_gather_nxv4i16:163; CHECK: // %bb.0:164; CHECK-NEXT: ld1h { z0.s }, p0/z, [x0, z0.s, uxtw]165; CHECK-NEXT: ret166 %offsets.zext = zext <vscale x 4 x i32> %offsets to <vscale x 4 x i64>167 %byte_ptrs = getelementptr i8, ptr %base, <vscale x 4 x i64> %offsets.zext168 %ptrs = bitcast <vscale x 4 x ptr> %byte_ptrs to <vscale x 4 x ptr>169 %vals = call <vscale x 4 x i16> @llvm.masked.gather.nxv4i16(<vscale x 4 x ptr> %ptrs, i32 2, <vscale x 4 x i1> %mask, <vscale x 4 x i16> poison)170 %vals.zext = zext <vscale x 4 x i16> %vals to <vscale x 4 x i32>171 ret <vscale x 4 x i32> %vals.zext172}173 174define <vscale x 4 x i32> @masked_gather_nxv4i32(ptr %base, <vscale x 4 x i32> %offsets, <vscale x 4 x i1> %mask) {175; CHECK-LABEL: masked_gather_nxv4i32:176; CHECK: // %bb.0:177; CHECK-NEXT: ld1w { z0.s }, p0/z, [x0, z0.s, uxtw]178; CHECK-NEXT: ret179 %offsets.zext = zext <vscale x 4 x i32> %offsets to <vscale x 4 x i64>180 %byte_ptrs = getelementptr i8, ptr %base, <vscale x 4 x i64> %offsets.zext181 %ptrs = bitcast <vscale x 4 x ptr> %byte_ptrs to <vscale x 4 x ptr>182 %vals = call <vscale x 4 x i32> @llvm.masked.gather.nxv4i32(<vscale x 4 x ptr> %ptrs, i32 4, <vscale x 4 x i1> %mask, <vscale x 4 x i32> poison)183 ret <vscale x 4 x i32> %vals184}185 186define <vscale x 4 x half> @masked_gather_nxv4f16(ptr %base, <vscale x 4 x i32> %offsets, <vscale x 4 x i1> %mask) {187; CHECK-LABEL: masked_gather_nxv4f16:188; CHECK: // %bb.0:189; CHECK-NEXT: ld1h { z0.s }, p0/z, [x0, z0.s, uxtw]190; CHECK-NEXT: ret191 %offsets.zext = zext <vscale x 4 x i32> %offsets to <vscale x 4 x i64>192 %byte_ptrs = getelementptr i8, ptr %base, <vscale x 4 x i64> %offsets.zext193 %ptrs = bitcast <vscale x 4 x ptr> %byte_ptrs to <vscale x 4 x ptr>194 %vals = call <vscale x 4 x half> @llvm.masked.gather.nxv4f16(<vscale x 4 x ptr> %ptrs, i32 2, <vscale x 4 x i1> %mask, <vscale x 4 x half> poison)195 ret <vscale x 4 x half> %vals196}197 198define <vscale x 4 x bfloat> @masked_gather_nxv4bf16(ptr %base, <vscale x 4 x i32> %offsets, <vscale x 4 x i1> %mask) #0 {199; CHECK-LABEL: masked_gather_nxv4bf16:200; CHECK: // %bb.0:201; CHECK-NEXT: ld1h { z0.s }, p0/z, [x0, z0.s, uxtw]202; CHECK-NEXT: ret203 %offsets.zext = zext <vscale x 4 x i32> %offsets to <vscale x 4 x i64>204 %byte_ptrs = getelementptr i8, ptr %base, <vscale x 4 x i64> %offsets.zext205 %ptrs = bitcast <vscale x 4 x ptr> %byte_ptrs to <vscale x 4 x ptr>206 %vals = call <vscale x 4 x bfloat> @llvm.masked.gather.nxv4bf16(<vscale x 4 x ptr> %ptrs, i32 2, <vscale x 4 x i1> %mask, <vscale x 4 x bfloat> poison)207 ret <vscale x 4 x bfloat> %vals208}209 210define <vscale x 4 x float> @masked_gather_nxv4f32(ptr %base, <vscale x 4 x i32> %offsets, <vscale x 4 x i1> %mask) {211; CHECK-LABEL: masked_gather_nxv4f32:212; CHECK: // %bb.0:213; CHECK-NEXT: ld1w { z0.s }, p0/z, [x0, z0.s, uxtw]214; CHECK-NEXT: ret215 %offsets.zext = zext <vscale x 4 x i32> %offsets to <vscale x 4 x i64>216 %byte_ptrs = getelementptr i8, ptr %base, <vscale x 4 x i64> %offsets.zext217 %ptrs = bitcast <vscale x 4 x ptr> %byte_ptrs to <vscale x 4 x ptr>218 %vals = call <vscale x 4 x float> @llvm.masked.gather.nxv4f32(<vscale x 4 x ptr> %ptrs, i32 4, <vscale x 4 x i1> %mask, <vscale x 4 x float> poison)219 ret <vscale x 4 x float> %vals220}221 222define <vscale x 4 x i32> @masked_sgather_nxv4i8(ptr %base, <vscale x 4 x i32> %offsets, <vscale x 4 x i1> %mask) {223; CHECK-LABEL: masked_sgather_nxv4i8:224; CHECK: // %bb.0:225; CHECK-NEXT: ld1sb { z0.s }, p0/z, [x0, z0.s, uxtw]226; CHECK-NEXT: ret227 %offsets.zext = zext <vscale x 4 x i32> %offsets to <vscale x 4 x i64>228 %ptrs = getelementptr i8, ptr %base, <vscale x 4 x i64> %offsets.zext229 %vals = call <vscale x 4 x i8> @llvm.masked.gather.nxv4i8(<vscale x 4 x ptr> %ptrs, i32 1, <vscale x 4 x i1> %mask, <vscale x 4 x i8> poison)230 %vals.sext = sext <vscale x 4 x i8> %vals to <vscale x 4 x i32>231 ret <vscale x 4 x i32> %vals.sext232}233 234define <vscale x 4 x i32> @masked_sgather_nxv4i16(ptr %base, <vscale x 4 x i32> %offsets, <vscale x 4 x i1> %mask) {235; CHECK-LABEL: masked_sgather_nxv4i16:236; CHECK: // %bb.0:237; CHECK-NEXT: ld1sh { z0.s }, p0/z, [x0, z0.s, uxtw]238; CHECK-NEXT: ret239 %offsets.zext = zext <vscale x 4 x i32> %offsets to <vscale x 4 x i64>240 %byte_ptrs = getelementptr i8, ptr %base, <vscale x 4 x i64> %offsets.zext241 %ptrs = bitcast <vscale x 4 x ptr> %byte_ptrs to <vscale x 4 x ptr>242 %vals = call <vscale x 4 x i16> @llvm.masked.gather.nxv4i16(<vscale x 4 x ptr> %ptrs, i32 2, <vscale x 4 x i1> %mask, <vscale x 4 x i16> poison)243 %vals.sext = sext <vscale x 4 x i16> %vals to <vscale x 4 x i32>244 ret <vscale x 4 x i32> %vals.sext245}246 247declare <vscale x 2 x i8> @llvm.masked.gather.nxv2i8(<vscale x 2 x ptr>, i32, <vscale x 2 x i1>, <vscale x 2 x i8>)248declare <vscale x 2 x i16> @llvm.masked.gather.nxv2i16(<vscale x 2 x ptr>, i32, <vscale x 2 x i1>, <vscale x 2 x i16>)249declare <vscale x 2 x i32> @llvm.masked.gather.nxv2i32(<vscale x 2 x ptr>, i32, <vscale x 2 x i1>, <vscale x 2 x i32>)250declare <vscale x 2 x i64> @llvm.masked.gather.nxv2i64(<vscale x 2 x ptr>, i32, <vscale x 2 x i1>, <vscale x 2 x i64>)251declare <vscale x 2 x half> @llvm.masked.gather.nxv2f16(<vscale x 2 x ptr>, i32, <vscale x 2 x i1>, <vscale x 2 x half>)252declare <vscale x 2 x bfloat> @llvm.masked.gather.nxv2bf16(<vscale x 2 x ptr>, i32, <vscale x 2 x i1>, <vscale x 2 x bfloat>)253declare <vscale x 2 x float> @llvm.masked.gather.nxv2f32(<vscale x 2 x ptr>, i32, <vscale x 2 x i1>, <vscale x 2 x float>)254declare <vscale x 2 x double> @llvm.masked.gather.nxv2f64(<vscale x 2 x ptr>, i32, <vscale x 2 x i1>, <vscale x 2 x double>)255 256declare <vscale x 4 x i8> @llvm.masked.gather.nxv4i8(<vscale x 4 x ptr>, i32, <vscale x 4 x i1>, <vscale x 4 x i8>)257declare <vscale x 4 x i16> @llvm.masked.gather.nxv4i16(<vscale x 4 x ptr>, i32, <vscale x 4 x i1>, <vscale x 4 x i16>)258declare <vscale x 4 x i32> @llvm.masked.gather.nxv4i32(<vscale x 4 x ptr>, i32, <vscale x 4 x i1>, <vscale x 4 x i32>)259declare <vscale x 4 x half> @llvm.masked.gather.nxv4f16(<vscale x 4 x ptr>, i32, <vscale x 4 x i1>, <vscale x 4 x half>)260declare <vscale x 4 x bfloat> @llvm.masked.gather.nxv4bf16(<vscale x 4 x ptr>, i32, <vscale x 4 x i1>, <vscale x 4 x bfloat>)261declare <vscale x 4 x float> @llvm.masked.gather.nxv4f32(<vscale x 4 x ptr>, i32, <vscale x 4 x i1>, <vscale x 4 x float>)262attributes #0 = { "target-features"="+sve,+bf16" }263