244 lines · plain
1; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py2; RUN: llc -mtriple=aarch64--linux-gnu -mattr=+sve -aarch64-enable-mgather-combine=0 < %s | FileCheck %s3; RUN: llc -mtriple=aarch64--linux-gnu -mattr=+sve -aarch64-enable-mgather-combine=1 < %s | FileCheck %s4 5;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;6; unscaled unpacked 32-bit offsets7;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;8 9define <vscale x 2 x i64> @masked_gather_nxv2i8(ptr %base, <vscale x 2 x i32> %offsets, <vscale x 2 x i1> %mask) {10; CHECK-LABEL: masked_gather_nxv2i8:11; CHECK: // %bb.0:12; CHECK-NEXT: ld1b { z0.d }, p0/z, [x0, z0.d, sxtw]13; CHECK-NEXT: ret14 %ptrs = getelementptr i8, ptr %base, <vscale x 2 x i32> %offsets15 %vals = call <vscale x 2 x i8> @llvm.masked.gather.nxv2i8(<vscale x 2 x ptr> %ptrs, i32 1, <vscale x 2 x i1> %mask, <vscale x 2 x i8> poison)16 %vals.zext = zext <vscale x 2 x i8> %vals to <vscale x 2 x i64>17 ret <vscale x 2 x i64> %vals.zext18}19 20define <vscale x 2 x i64> @masked_gather_nxv2i16(ptr %base, <vscale x 2 x i32> %offsets, <vscale x 2 x i1> %mask) {21; CHECK-LABEL: masked_gather_nxv2i16:22; CHECK: // %bb.0:23; CHECK-NEXT: ld1h { z0.d }, p0/z, [x0, z0.d, sxtw]24; CHECK-NEXT: ret25 %byte_ptrs = getelementptr i8, ptr %base, <vscale x 2 x i32> %offsets26 %ptrs = bitcast <vscale x 2 x ptr> %byte_ptrs to <vscale x 2 x ptr>27 %vals = call <vscale x 2 x i16> @llvm.masked.gather.nxv2i16(<vscale x 2 x ptr> %ptrs, i32 2, <vscale x 2 x i1> %mask, <vscale x 2 x i16> poison)28 %vals.zext = zext <vscale x 2 x i16> %vals to <vscale x 2 x i64>29 ret <vscale x 2 x i64> %vals.zext30}31 32define <vscale x 2 x i64> @masked_gather_nxv2i32(ptr %base, <vscale x 2 x i32> %offsets, <vscale x 2 x i1> %mask) {33; CHECK-LABEL: masked_gather_nxv2i32:34; CHECK: // %bb.0:35; CHECK-NEXT: ld1w { z0.d }, p0/z, [x0, z0.d, sxtw]36; CHECK-NEXT: ret37 %byte_ptrs = getelementptr i8, ptr %base, <vscale x 2 x i32> %offsets38 %ptrs = bitcast <vscale x 2 x ptr> %byte_ptrs to <vscale x 2 x ptr>39 %vals = call <vscale x 2 x i32> @llvm.masked.gather.nxv2i32(<vscale x 2 x ptr> %ptrs, i32 4, <vscale x 2 x i1> %mask, <vscale x 2 x i32> poison)40 %vals.zext = zext <vscale x 2 x i32> %vals to <vscale x 2 x i64>41 ret <vscale x 2 x i64> %vals.zext42}43 44define <vscale x 2 x i64> @masked_gather_nxv2i64(ptr %base, <vscale x 2 x i32> %offsets, <vscale x 2 x i1> %mask) {45; CHECK-LABEL: masked_gather_nxv2i64:46; CHECK: // %bb.0:47; CHECK-NEXT: ld1d { z0.d }, p0/z, [x0, z0.d, sxtw]48; CHECK-NEXT: ret49 %byte_ptrs = getelementptr i8, ptr %base, <vscale x 2 x i32> %offsets50 %ptrs = bitcast <vscale x 2 x ptr> %byte_ptrs to <vscale x 2 x ptr>51 %vals = call <vscale x 2 x i64> @llvm.masked.gather.nxv2i64(<vscale x 2 x ptr> %ptrs, i32 8, <vscale x 2 x i1> %mask, <vscale x 2 x i64> poison)52 ret <vscale x 2 x i64> %vals53}54 55define <vscale x 2 x half> @masked_gather_nxv2f16(ptr %base, <vscale x 2 x i32> %offsets, <vscale x 2 x i1> %mask) {56; CHECK-LABEL: masked_gather_nxv2f16:57; CHECK: // %bb.0:58; CHECK-NEXT: ld1h { z0.d }, p0/z, [x0, z0.d, sxtw]59; CHECK-NEXT: ret60 %byte_ptrs = getelementptr i8, ptr %base, <vscale x 2 x i32> %offsets61 %ptrs = bitcast <vscale x 2 x ptr> %byte_ptrs to <vscale x 2 x ptr>62 %vals = call <vscale x 2 x half> @llvm.masked.gather.nxv2f16(<vscale x 2 x ptr> %ptrs, i32 2, <vscale x 2 x i1> %mask, <vscale x 2 x half> poison)63 ret <vscale x 2 x half> %vals64}65 66define <vscale x 2 x bfloat> @masked_gather_nxv2bf16(ptr %base, <vscale x 2 x i32> %offsets, <vscale x 2 x i1> %mask) #0 {67; CHECK-LABEL: masked_gather_nxv2bf16:68; CHECK: // %bb.0:69; CHECK-NEXT: ld1h { z0.d }, p0/z, [x0, z0.d, sxtw]70; CHECK-NEXT: ret71 %byte_ptrs = getelementptr i8, ptr %base, <vscale x 2 x i32> %offsets72 %ptrs = bitcast <vscale x 2 x ptr> %byte_ptrs to <vscale x 2 x ptr>73 %vals = call <vscale x 2 x bfloat> @llvm.masked.gather.nxv2bf16(<vscale x 2 x ptr> %ptrs, i32 2, <vscale x 2 x i1> %mask, <vscale x 2 x bfloat> poison)74 ret <vscale x 2 x bfloat> %vals75}76 77define <vscale x 2 x float> @masked_gather_nxv2f32(ptr %base, <vscale x 2 x i32> %offsets, <vscale x 2 x i1> %mask) {78; CHECK-LABEL: masked_gather_nxv2f32:79; CHECK: // %bb.0:80; CHECK-NEXT: ld1w { z0.d }, p0/z, [x0, z0.d, sxtw]81; CHECK-NEXT: ret82 %byte_ptrs = getelementptr i8, ptr %base, <vscale x 2 x i32> %offsets83 %ptrs = bitcast <vscale x 2 x ptr> %byte_ptrs to <vscale x 2 x ptr>84 %vals = call <vscale x 2 x float> @llvm.masked.gather.nxv2f32(<vscale x 2 x ptr> %ptrs, i32 4, <vscale x 2 x i1> %mask, <vscale x 2 x float> poison)85 ret <vscale x 2 x float> %vals86}87 88define <vscale x 2 x double> @masked_gather_nxv2f64(ptr %base, <vscale x 2 x i32> %offsets, <vscale x 2 x i1> %mask) {89; CHECK-LABEL: masked_gather_nxv2f64:90; CHECK: // %bb.0:91; CHECK-NEXT: ld1d { z0.d }, p0/z, [x0, z0.d, sxtw]92; CHECK-NEXT: ret93 %byte_ptrs = getelementptr i8, ptr %base, <vscale x 2 x i32> %offsets94 %ptrs = bitcast <vscale x 2 x ptr> %byte_ptrs to <vscale x 2 x ptr>95 %vals = call <vscale x 2 x double> @llvm.masked.gather.nxv2f64(<vscale x 2 x ptr> %ptrs, i32 8, <vscale x 2 x i1> %mask, <vscale x 2 x double> poison)96 ret <vscale x 2 x double> %vals97}98 99define <vscale x 2 x i64> @masked_sgather_nxv2i8(ptr %base, <vscale x 2 x i32> %offsets, <vscale x 2 x i1> %mask) {100; CHECK-LABEL: masked_sgather_nxv2i8:101; CHECK: // %bb.0:102; CHECK-NEXT: ld1sb { z0.d }, p0/z, [x0, z0.d, sxtw]103; CHECK-NEXT: ret104 %ptrs = getelementptr i8, ptr %base, <vscale x 2 x i32> %offsets105 %vals = call <vscale x 2 x i8> @llvm.masked.gather.nxv2i8(<vscale x 2 x ptr> %ptrs, i32 1, <vscale x 2 x i1> %mask, <vscale x 2 x i8> poison)106 %vals.sext = sext <vscale x 2 x i8> %vals to <vscale x 2 x i64>107 ret <vscale x 2 x i64> %vals.sext108}109 110define <vscale x 2 x i64> @masked_sgather_nxv2i16(ptr %base, <vscale x 2 x i32> %offsets, <vscale x 2 x i1> %mask) {111; CHECK-LABEL: masked_sgather_nxv2i16:112; CHECK: // %bb.0:113; CHECK-NEXT: ld1sh { z0.d }, p0/z, [x0, z0.d, sxtw]114; CHECK-NEXT: ret115 %byte_ptrs = getelementptr i8, ptr %base, <vscale x 2 x i32> %offsets116 %ptrs = bitcast <vscale x 2 x ptr> %byte_ptrs to <vscale x 2 x ptr>117 %vals = call <vscale x 2 x i16> @llvm.masked.gather.nxv2i16(<vscale x 2 x ptr> %ptrs, i32 2, <vscale x 2 x i1> %mask, <vscale x 2 x i16> poison)118 %vals.sext = sext <vscale x 2 x i16> %vals to <vscale x 2 x i64>119 ret <vscale x 2 x i64> %vals.sext120}121 122define <vscale x 2 x i64> @masked_sgather_nxv2i32(ptr %base, <vscale x 2 x i32> %offsets, <vscale x 2 x i1> %mask) {123; CHECK-LABEL: masked_sgather_nxv2i32:124; CHECK: // %bb.0:125; CHECK-NEXT: ld1sw { z0.d }, p0/z, [x0, z0.d, sxtw]126; CHECK-NEXT: ret127 %byte_ptrs = getelementptr i8, ptr %base, <vscale x 2 x i32> %offsets128 %ptrs = bitcast <vscale x 2 x ptr> %byte_ptrs to <vscale x 2 x ptr>129 %vals = call <vscale x 2 x i32> @llvm.masked.gather.nxv2i32(<vscale x 2 x ptr> %ptrs, i32 4, <vscale x 2 x i1> %mask, <vscale x 2 x i32> poison)130 %vals.sext = sext <vscale x 2 x i32> %vals to <vscale x 2 x i64>131 ret <vscale x 2 x i64> %vals.sext132}133 134;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;135; unscaled packed 32-bit offsets136;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;137 138define <vscale x 4 x i32> @masked_gather_nxv4i8(ptr %base, <vscale x 4 x i32> %offsets, <vscale x 4 x i1> %mask) {139; CHECK-LABEL: masked_gather_nxv4i8:140; CHECK: // %bb.0:141; CHECK-NEXT: ld1b { z0.s }, p0/z, [x0, z0.s, sxtw]142; CHECK-NEXT: ret143 %ptrs = getelementptr i8, ptr %base, <vscale x 4 x i32> %offsets144 %vals = call <vscale x 4 x i8> @llvm.masked.gather.nxv4i8(<vscale x 4 x ptr> %ptrs, i32 1, <vscale x 4 x i1> %mask, <vscale x 4 x i8> poison)145 %vals.zext = zext <vscale x 4 x i8> %vals to <vscale x 4 x i32>146 ret <vscale x 4 x i32> %vals.zext147}148 149define <vscale x 4 x i32> @masked_gather_nxv4i16(ptr %base, <vscale x 4 x i32> %offsets, <vscale x 4 x i1> %mask) {150; CHECK-LABEL: masked_gather_nxv4i16:151; CHECK: // %bb.0:152; CHECK-NEXT: ld1h { z0.s }, p0/z, [x0, z0.s, sxtw]153; CHECK-NEXT: ret154 %byte_ptrs = getelementptr i8, ptr %base, <vscale x 4 x i32> %offsets155 %ptrs = bitcast <vscale x 4 x ptr> %byte_ptrs to <vscale x 4 x ptr>156 %vals = call <vscale x 4 x i16> @llvm.masked.gather.nxv4i16(<vscale x 4 x ptr> %ptrs, i32 2, <vscale x 4 x i1> %mask, <vscale x 4 x i16> poison)157 %vals.zext = zext <vscale x 4 x i16> %vals to <vscale x 4 x i32>158 ret <vscale x 4 x i32> %vals.zext159}160 161define <vscale x 4 x i32> @masked_gather_nxv4i32(ptr %base, <vscale x 4 x i32> %offsets, <vscale x 4 x i1> %mask) {162; CHECK-LABEL: masked_gather_nxv4i32:163; CHECK: // %bb.0:164; CHECK-NEXT: ld1w { z0.s }, p0/z, [x0, z0.s, sxtw]165; CHECK-NEXT: ret166 %byte_ptrs = getelementptr i8, ptr %base, <vscale x 4 x i32> %offsets167 %ptrs = bitcast <vscale x 4 x ptr> %byte_ptrs to <vscale x 4 x ptr>168 %vals = call <vscale x 4 x i32> @llvm.masked.gather.nxv4i32(<vscale x 4 x ptr> %ptrs, i32 4, <vscale x 4 x i1> %mask, <vscale x 4 x i32> poison)169 ret <vscale x 4 x i32> %vals170}171 172define <vscale x 4 x half> @masked_gather_nxv4f16(ptr %base, <vscale x 4 x i32> %offsets, <vscale x 4 x i1> %mask) {173; CHECK-LABEL: masked_gather_nxv4f16:174; CHECK: // %bb.0:175; CHECK-NEXT: ld1h { z0.s }, p0/z, [x0, z0.s, sxtw]176; CHECK-NEXT: ret177 %byte_ptrs = getelementptr i8, ptr %base, <vscale x 4 x i32> %offsets178 %ptrs = bitcast <vscale x 4 x ptr> %byte_ptrs to <vscale x 4 x ptr>179 %vals = call <vscale x 4 x half> @llvm.masked.gather.nxv4f16(<vscale x 4 x ptr> %ptrs, i32 2, <vscale x 4 x i1> %mask, <vscale x 4 x half> poison)180 ret <vscale x 4 x half> %vals181}182 183define <vscale x 4 x bfloat> @masked_gather_nxv4bf16(ptr %base, <vscale x 4 x i32> %offsets, <vscale x 4 x i1> %mask) #0 {184; CHECK-LABEL: masked_gather_nxv4bf16:185; CHECK: // %bb.0:186; CHECK-NEXT: ld1h { z0.s }, p0/z, [x0, z0.s, sxtw]187; CHECK-NEXT: ret188 %byte_ptrs = getelementptr i8, ptr %base, <vscale x 4 x i32> %offsets189 %ptrs = bitcast <vscale x 4 x ptr> %byte_ptrs to <vscale x 4 x ptr>190 %vals = call <vscale x 4 x bfloat> @llvm.masked.gather.nxv4bf16(<vscale x 4 x ptr> %ptrs, i32 2, <vscale x 4 x i1> %mask, <vscale x 4 x bfloat> poison)191 ret <vscale x 4 x bfloat> %vals192}193 194define <vscale x 4 x float> @masked_gather_nxv4f32(ptr %base, <vscale x 4 x i32> %offsets, <vscale x 4 x i1> %mask) {195; CHECK-LABEL: masked_gather_nxv4f32:196; CHECK: // %bb.0:197; CHECK-NEXT: ld1w { z0.s }, p0/z, [x0, z0.s, sxtw]198; CHECK-NEXT: ret199 %byte_ptrs = getelementptr i8, ptr %base, <vscale x 4 x i32> %offsets200 %ptrs = bitcast <vscale x 4 x ptr> %byte_ptrs to <vscale x 4 x ptr>201 %vals = call <vscale x 4 x float> @llvm.masked.gather.nxv4f32(<vscale x 4 x ptr> %ptrs, i32 4, <vscale x 4 x i1> %mask, <vscale x 4 x float> poison)202 ret <vscale x 4 x float> %vals203}204 205define <vscale x 4 x i32> @masked_sgather_nxv4i8(ptr %base, <vscale x 4 x i32> %offsets, <vscale x 4 x i1> %mask) {206; CHECK-LABEL: masked_sgather_nxv4i8:207; CHECK: // %bb.0:208; CHECK-NEXT: ld1sb { z0.s }, p0/z, [x0, z0.s, sxtw]209; CHECK-NEXT: ret210 %ptrs = getelementptr i8, ptr %base, <vscale x 4 x i32> %offsets211 %vals = call <vscale x 4 x i8> @llvm.masked.gather.nxv4i8(<vscale x 4 x ptr> %ptrs, i32 1, <vscale x 4 x i1> %mask, <vscale x 4 x i8> poison)212 %vals.sext = sext <vscale x 4 x i8> %vals to <vscale x 4 x i32>213 ret <vscale x 4 x i32> %vals.sext214}215 216define <vscale x 4 x i32> @masked_sgather_nxv4i16(ptr %base, <vscale x 4 x i32> %offsets, <vscale x 4 x i1> %mask) {217; CHECK-LABEL: masked_sgather_nxv4i16:218; CHECK: // %bb.0:219; CHECK-NEXT: ld1sh { z0.s }, p0/z, [x0, z0.s, sxtw]220; CHECK-NEXT: ret221 %byte_ptrs = getelementptr i8, ptr %base, <vscale x 4 x i32> %offsets222 %ptrs = bitcast <vscale x 4 x ptr> %byte_ptrs to <vscale x 4 x ptr>223 %vals = call <vscale x 4 x i16> @llvm.masked.gather.nxv4i16(<vscale x 4 x ptr> %ptrs, i32 2, <vscale x 4 x i1> %mask, <vscale x 4 x i16> poison)224 %vals.sext = sext <vscale x 4 x i16> %vals to <vscale x 4 x i32>225 ret <vscale x 4 x i32> %vals.sext226}227 228declare <vscale x 2 x i8> @llvm.masked.gather.nxv2i8(<vscale x 2 x ptr>, i32, <vscale x 2 x i1>, <vscale x 2 x i8>)229declare <vscale x 2 x i16> @llvm.masked.gather.nxv2i16(<vscale x 2 x ptr>, i32, <vscale x 2 x i1>, <vscale x 2 x i16>)230declare <vscale x 2 x i32> @llvm.masked.gather.nxv2i32(<vscale x 2 x ptr>, i32, <vscale x 2 x i1>, <vscale x 2 x i32>)231declare <vscale x 2 x i64> @llvm.masked.gather.nxv2i64(<vscale x 2 x ptr>, i32, <vscale x 2 x i1>, <vscale x 2 x i64>)232declare <vscale x 2 x half> @llvm.masked.gather.nxv2f16(<vscale x 2 x ptr>, i32, <vscale x 2 x i1>, <vscale x 2 x half>)233declare <vscale x 2 x bfloat> @llvm.masked.gather.nxv2bf16(<vscale x 2 x ptr>, i32, <vscale x 2 x i1>, <vscale x 2 x bfloat>)234declare <vscale x 2 x float> @llvm.masked.gather.nxv2f32(<vscale x 2 x ptr>, i32, <vscale x 2 x i1>, <vscale x 2 x float>)235declare <vscale x 2 x double> @llvm.masked.gather.nxv2f64(<vscale x 2 x ptr>, i32, <vscale x 2 x i1>, <vscale x 2 x double>)236 237declare <vscale x 4 x i8> @llvm.masked.gather.nxv4i8(<vscale x 4 x ptr>, i32, <vscale x 4 x i1>, <vscale x 4 x i8>)238declare <vscale x 4 x i16> @llvm.masked.gather.nxv4i16(<vscale x 4 x ptr>, i32, <vscale x 4 x i1>, <vscale x 4 x i16>)239declare <vscale x 4 x i32> @llvm.masked.gather.nxv4i32(<vscale x 4 x ptr>, i32, <vscale x 4 x i1>, <vscale x 4 x i32>)240declare <vscale x 4 x half> @llvm.masked.gather.nxv4f16(<vscale x 4 x ptr>, i32, <vscale x 4 x i1>, <vscale x 4 x half>)241declare <vscale x 4 x bfloat> @llvm.masked.gather.nxv4bf16(<vscale x 4 x ptr>, i32, <vscale x 4 x i1>, <vscale x 4 x bfloat>)242declare <vscale x 4 x float> @llvm.masked.gather.nxv4f32(<vscale x 4 x ptr>, i32, <vscale x 4 x i1>, <vscale x 4 x float>)243attributes #0 = { "target-features"="+sve,+bf16" }244