183 lines · plain
1; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py2; RUN: llc -mtriple=aarch64--linux-gnu -mattr=+sve -aarch64-enable-mgather-combine=0 < %s | FileCheck %s3; RUN: llc -mtriple=aarch64--linux-gnu -mattr=+sve -aarch64-enable-mgather-combine=1 < %s | FileCheck %s4 5;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;6; unscaled unpacked 32-bit offsets7;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;8 9define <vscale x 2 x i64> @masked_gather_nxv2i16(ptr %base, <vscale x 2 x i32> %offsets, <vscale x 2 x i1> %mask) {10; CHECK-LABEL: masked_gather_nxv2i16:11; CHECK: // %bb.0:12; CHECK-NEXT: ld1h { z0.d }, p0/z, [x0, z0.d, sxtw #1]13; CHECK-NEXT: ret14 %ptrs = getelementptr i16, ptr %base, <vscale x 2 x i32> %offsets15 %vals = call <vscale x 2 x i16> @llvm.masked.gather.nxv2i16(<vscale x 2 x ptr> %ptrs, i32 2, <vscale x 2 x i1> %mask, <vscale x 2 x i16> poison)16 %vals.zext = zext <vscale x 2 x i16> %vals to <vscale x 2 x i64>17 ret <vscale x 2 x i64> %vals.zext18}19 20define <vscale x 2 x i64> @masked_gather_nxv2i32(ptr %base, <vscale x 2 x i32> %offsets, <vscale x 2 x i1> %mask) {21; CHECK-LABEL: masked_gather_nxv2i32:22; CHECK: // %bb.0:23; CHECK-NEXT: ld1w { z0.d }, p0/z, [x0, z0.d, sxtw #2]24; CHECK-NEXT: ret25 %ptrs = getelementptr i32, ptr %base, <vscale x 2 x i32> %offsets26 %vals = call <vscale x 2 x i32> @llvm.masked.gather.nxv2i32(<vscale x 2 x ptr> %ptrs, i32 4, <vscale x 2 x i1> %mask, <vscale x 2 x i32> poison)27 %vals.zext = zext <vscale x 2 x i32> %vals to <vscale x 2 x i64>28 ret <vscale x 2 x i64> %vals.zext29}30 31define <vscale x 2 x i64> @masked_gather_nxv2i64(ptr %base, <vscale x 2 x i32> %offsets, <vscale x 2 x i1> %mask) {32; CHECK-LABEL: masked_gather_nxv2i64:33; CHECK: // %bb.0:34; CHECK-NEXT: ld1d { z0.d }, p0/z, [x0, z0.d, sxtw #3]35; CHECK-NEXT: ret36 %ptrs = getelementptr i64, ptr %base, <vscale x 2 x i32> %offsets37 %vals = call <vscale x 2 x i64> @llvm.masked.gather.nxv2i64(<vscale x 2 x ptr> %ptrs, i32 8, <vscale x 2 x i1> %mask, <vscale x 2 x i64> poison)38 ret <vscale x 2 x i64> %vals39}40 41define <vscale x 2 x half> @masked_gather_nxv2f16(ptr %base, <vscale x 2 x i32> %offsets, <vscale x 2 x i1> %mask) {42; CHECK-LABEL: masked_gather_nxv2f16:43; CHECK: // %bb.0:44; CHECK-NEXT: ld1h { z0.d }, p0/z, [x0, z0.d, sxtw #1]45; CHECK-NEXT: ret46 %ptrs = getelementptr half, ptr %base, <vscale x 2 x i32> %offsets47 %vals = call <vscale x 2 x half> @llvm.masked.gather.nxv2f16(<vscale x 2 x ptr> %ptrs, i32 2, <vscale x 2 x i1> %mask, <vscale x 2 x half> poison)48 ret <vscale x 2 x half> %vals49}50 51define <vscale x 2 x bfloat> @masked_gather_nxv2bf16(ptr %base, <vscale x 2 x i32> %offsets, <vscale x 2 x i1> %mask) #0 {52; CHECK-LABEL: masked_gather_nxv2bf16:53; CHECK: // %bb.0:54; CHECK-NEXT: ld1h { z0.d }, p0/z, [x0, z0.d, sxtw #1]55; CHECK-NEXT: ret56 %ptrs = getelementptr bfloat, ptr %base, <vscale x 2 x i32> %offsets57 %vals = call <vscale x 2 x bfloat> @llvm.masked.gather.nxv2bf16(<vscale x 2 x ptr> %ptrs, i32 2, <vscale x 2 x i1> %mask, <vscale x 2 x bfloat> poison)58 ret <vscale x 2 x bfloat> %vals59}60 61define <vscale x 2 x float> @masked_gather_nxv2f32(ptr %base, <vscale x 2 x i32> %offsets, <vscale x 2 x i1> %mask) {62; CHECK-LABEL: masked_gather_nxv2f32:63; CHECK: // %bb.0:64; CHECK-NEXT: ld1w { z0.d }, p0/z, [x0, z0.d, sxtw #2]65; CHECK-NEXT: ret66 %ptrs = getelementptr float, ptr %base, <vscale x 2 x i32> %offsets67 %vals = call <vscale x 2 x float> @llvm.masked.gather.nxv2f32(<vscale x 2 x ptr> %ptrs, i32 4, <vscale x 2 x i1> %mask, <vscale x 2 x float> poison)68 ret <vscale x 2 x float> %vals69}70 71define <vscale x 2 x double> @masked_gather_nxv2f64(ptr %base, <vscale x 2 x i32> %offsets, <vscale x 2 x i1> %mask) {72; CHECK-LABEL: masked_gather_nxv2f64:73; CHECK: // %bb.0:74; CHECK-NEXT: ld1d { z0.d }, p0/z, [x0, z0.d, sxtw #3]75; CHECK-NEXT: ret76 %ptrs = getelementptr double, ptr %base, <vscale x 2 x i32> %offsets77 %vals = call <vscale x 2 x double> @llvm.masked.gather.nxv2f64(<vscale x 2 x ptr> %ptrs, i32 8, <vscale x 2 x i1> %mask, <vscale x 2 x double> poison)78 ret <vscale x 2 x double> %vals79}80 81define <vscale x 2 x i64> @masked_sgather_nxv2i16(ptr %base, <vscale x 2 x i32> %offsets, <vscale x 2 x i1> %mask) {82; CHECK-LABEL: masked_sgather_nxv2i16:83; CHECK: // %bb.0:84; CHECK-NEXT: ld1sh { z0.d }, p0/z, [x0, z0.d, sxtw #1]85; CHECK-NEXT: ret86 %ptrs = getelementptr i16, ptr %base, <vscale x 2 x i32> %offsets87 %vals = call <vscale x 2 x i16> @llvm.masked.gather.nxv2i16(<vscale x 2 x ptr> %ptrs, i32 2, <vscale x 2 x i1> %mask, <vscale x 2 x i16> poison)88 %vals.sext = sext <vscale x 2 x i16> %vals to <vscale x 2 x i64>89 ret <vscale x 2 x i64> %vals.sext90}91 92define <vscale x 2 x i64> @masked_sgather_nxv2i32(ptr %base, <vscale x 2 x i32> %offsets, <vscale x 2 x i1> %mask) {93; CHECK-LABEL: masked_sgather_nxv2i32:94; CHECK: // %bb.0:95; CHECK-NEXT: ld1sw { z0.d }, p0/z, [x0, z0.d, sxtw #2]96; CHECK-NEXT: ret97 %ptrs = getelementptr i32, ptr %base, <vscale x 2 x i32> %offsets98 %vals = call <vscale x 2 x i32> @llvm.masked.gather.nxv2i32(<vscale x 2 x ptr> %ptrs, i32 4, <vscale x 2 x i1> %mask, <vscale x 2 x i32> poison)99 %vals.sext = sext <vscale x 2 x i32> %vals to <vscale x 2 x i64>100 ret <vscale x 2 x i64> %vals.sext101}102 103;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;104; unscaled packed 32-bit offsets105;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;106 107define <vscale x 4 x i32> @masked_gather_nxv4i16(ptr %base, <vscale x 4 x i32> %offsets, <vscale x 4 x i1> %mask) {108; CHECK-LABEL: masked_gather_nxv4i16:109; CHECK: // %bb.0:110; CHECK-NEXT: ld1h { z0.s }, p0/z, [x0, z0.s, sxtw #1]111; CHECK-NEXT: ret112 %ptrs = getelementptr i16, ptr %base, <vscale x 4 x i32> %offsets113 %vals = call <vscale x 4 x i16> @llvm.masked.gather.nxv4i16(<vscale x 4 x ptr> %ptrs, i32 2, <vscale x 4 x i1> %mask, <vscale x 4 x i16> poison)114 %vals.zext = zext <vscale x 4 x i16> %vals to <vscale x 4 x i32>115 ret <vscale x 4 x i32> %vals.zext116}117 118define <vscale x 4 x i32> @masked_gather_nxv4i32(ptr %base, <vscale x 4 x i32> %offsets, <vscale x 4 x i1> %mask) {119; CHECK-LABEL: masked_gather_nxv4i32:120; CHECK: // %bb.0:121; CHECK-NEXT: ld1w { z0.s }, p0/z, [x0, z0.s, sxtw #2]122; CHECK-NEXT: ret123 %ptrs = getelementptr i32, ptr %base, <vscale x 4 x i32> %offsets124 %vals = call <vscale x 4 x i32> @llvm.masked.gather.nxv4i32(<vscale x 4 x ptr> %ptrs, i32 4, <vscale x 4 x i1> %mask, <vscale x 4 x i32> poison)125 ret <vscale x 4 x i32> %vals126}127 128define <vscale x 4 x half> @masked_gather_nxv4f16(ptr %base, <vscale x 4 x i32> %offsets, <vscale x 4 x i1> %mask) {129; CHECK-LABEL: masked_gather_nxv4f16:130; CHECK: // %bb.0:131; CHECK-NEXT: ld1h { z0.s }, p0/z, [x0, z0.s, sxtw #1]132; CHECK-NEXT: ret133 %ptrs = getelementptr half, ptr %base, <vscale x 4 x i32> %offsets134 %vals = call <vscale x 4 x half> @llvm.masked.gather.nxv4f16(<vscale x 4 x ptr> %ptrs, i32 2, <vscale x 4 x i1> %mask, <vscale x 4 x half> poison)135 ret <vscale x 4 x half> %vals136}137 138define <vscale x 4 x bfloat> @masked_gather_nxv4bf16(ptr %base, <vscale x 4 x i32> %offsets, <vscale x 4 x i1> %mask) #0 {139; CHECK-LABEL: masked_gather_nxv4bf16:140; CHECK: // %bb.0:141; CHECK-NEXT: ld1h { z0.s }, p0/z, [x0, z0.s, sxtw #1]142; CHECK-NEXT: ret143 %ptrs = getelementptr bfloat, ptr %base, <vscale x 4 x i32> %offsets144 %vals = call <vscale x 4 x bfloat> @llvm.masked.gather.nxv4bf16(<vscale x 4 x ptr> %ptrs, i32 2, <vscale x 4 x i1> %mask, <vscale x 4 x bfloat> poison)145 ret <vscale x 4 x bfloat> %vals146}147 148define <vscale x 4 x float> @masked_gather_nxv4f32(ptr %base, <vscale x 4 x i32> %offsets, <vscale x 4 x i1> %mask) {149; CHECK-LABEL: masked_gather_nxv4f32:150; CHECK: // %bb.0:151; CHECK-NEXT: ld1w { z0.s }, p0/z, [x0, z0.s, sxtw #2]152; CHECK-NEXT: ret153 %ptrs = getelementptr float, ptr %base, <vscale x 4 x i32> %offsets154 %vals = call <vscale x 4 x float> @llvm.masked.gather.nxv4f32(<vscale x 4 x ptr> %ptrs, i32 4, <vscale x 4 x i1> %mask, <vscale x 4 x float> poison)155 ret <vscale x 4 x float> %vals156}157 158define <vscale x 4 x i32> @masked_sgather_nxv4i16(ptr %base, <vscale x 4 x i32> %offsets, <vscale x 4 x i1> %mask) {159; CHECK-LABEL: masked_sgather_nxv4i16:160; CHECK: // %bb.0:161; CHECK-NEXT: ld1sh { z0.s }, p0/z, [x0, z0.s, sxtw #1]162; CHECK-NEXT: ret163 %ptrs = getelementptr i16, ptr %base, <vscale x 4 x i32> %offsets164 %vals = call <vscale x 4 x i16> @llvm.masked.gather.nxv4i16(<vscale x 4 x ptr> %ptrs, i32 2, <vscale x 4 x i1> %mask, <vscale x 4 x i16> poison)165 %vals.sext = sext <vscale x 4 x i16> %vals to <vscale x 4 x i32>166 ret <vscale x 4 x i32> %vals.sext167}168 169declare <vscale x 2 x i16> @llvm.masked.gather.nxv2i16(<vscale x 2 x ptr>, i32, <vscale x 2 x i1>, <vscale x 2 x i16>)170declare <vscale x 2 x i32> @llvm.masked.gather.nxv2i32(<vscale x 2 x ptr>, i32, <vscale x 2 x i1>, <vscale x 2 x i32>)171declare <vscale x 2 x i64> @llvm.masked.gather.nxv2i64(<vscale x 2 x ptr>, i32, <vscale x 2 x i1>, <vscale x 2 x i64>)172declare <vscale x 2 x half> @llvm.masked.gather.nxv2f16(<vscale x 2 x ptr>, i32, <vscale x 2 x i1>, <vscale x 2 x half>)173declare <vscale x 2 x bfloat> @llvm.masked.gather.nxv2bf16(<vscale x 2 x ptr>, i32, <vscale x 2 x i1>, <vscale x 2 x bfloat>)174declare <vscale x 2 x float> @llvm.masked.gather.nxv2f32(<vscale x 2 x ptr>, i32, <vscale x 2 x i1>, <vscale x 2 x float>)175declare <vscale x 2 x double> @llvm.masked.gather.nxv2f64(<vscale x 2 x ptr>, i32, <vscale x 2 x i1>, <vscale x 2 x double>)176 177declare <vscale x 4 x i16> @llvm.masked.gather.nxv4i16(<vscale x 4 x ptr>, i32, <vscale x 4 x i1>, <vscale x 4 x i16>)178declare <vscale x 4 x i32> @llvm.masked.gather.nxv4i32(<vscale x 4 x ptr>, i32, <vscale x 4 x i1>, <vscale x 4 x i32>)179declare <vscale x 4 x half> @llvm.masked.gather.nxv4f16(<vscale x 4 x ptr>, i32, <vscale x 4 x i1>, <vscale x 4 x half>)180declare <vscale x 4 x bfloat> @llvm.masked.gather.nxv4bf16(<vscale x 4 x ptr>, i32, <vscale x 4 x i1>, <vscale x 4 x bfloat>)181declare <vscale x 4 x float> @llvm.masked.gather.nxv4f32(<vscale x 4 x ptr>, i32, <vscale x 4 x i1>, <vscale x 4 x float>)182attributes #0 = { "target-features"="+sve,+bf16" }183