634 lines · plain
1; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py2; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sve,+bf16 < %s | FileCheck %s3 4;5; LD1RQB6;7 8define <vscale x 16 x i8> @ld1rqb_i8(<vscale x 16 x i1> %pred, ptr %addr) {9; CHECK-LABEL: ld1rqb_i8:10; CHECK: // %bb.0:11; CHECK-NEXT: ld1rqb { z0.b }, p0/z, [x0]12; CHECK-NEXT: ret13 %res = call <vscale x 16 x i8> @llvm.aarch64.sve.ld1rq.nxv16i8(<vscale x 16 x i1> %pred, ptr %addr)14 ret <vscale x 16 x i8> %res15}16 17define <vscale x 16 x i8> @ld1rqb_i8_imm(<vscale x 16 x i1> %pred, ptr %addr) {18; CHECK-LABEL: ld1rqb_i8_imm:19; CHECK: // %bb.0:20; CHECK-NEXT: ld1rqb { z0.b }, p0/z, [x0, #16]21; CHECK-NEXT: ret22 %ptr = getelementptr inbounds i8, ptr %addr, i8 1623 %res = call <vscale x 16 x i8> @llvm.aarch64.sve.ld1rq.nxv16i8(<vscale x 16 x i1> %pred, ptr %ptr)24 ret <vscale x 16 x i8> %res25}26 27define <vscale x 16 x i8> @ld1rqb_i8_scalar(<vscale x 16 x i1> %pred, ptr %addr, i64 %idx) {28; CHECK-LABEL: ld1rqb_i8_scalar:29; CHECK: // %bb.0:30; CHECK-NEXT: ld1rqb { z0.b }, p0/z, [x0, x1]31; CHECK-NEXT: ret32 %ptr = getelementptr inbounds i8, ptr %addr, i64 %idx33 %res = call <vscale x 16 x i8> @llvm.aarch64.sve.ld1rq.nxv16i8(<vscale x 16 x i1> %pred, ptr %ptr)34 ret <vscale x 16 x i8> %res35}36 37define <vscale x 16 x i8> @ld1rqb_i8_imm_lower_bound(<vscale x 16 x i1> %pred, ptr %addr) {38; CHECK-LABEL: ld1rqb_i8_imm_lower_bound:39; CHECK: // %bb.0:40; CHECK-NEXT: ld1rqb { z0.b }, p0/z, [x0, #-128]41; CHECK-NEXT: ret42 %ptr = getelementptr inbounds i8, ptr %addr, i8 -12843 %res = call <vscale x 16 x i8> @llvm.aarch64.sve.ld1rq.nxv16i8(<vscale x 16 x i1> %pred, ptr %ptr)44 ret <vscale x 16 x i8> %res45}46 47define <vscale x 16 x i8> @ld1rqb_i8_imm_upper_bound(<vscale x 16 x i1> %pred, ptr %addr) {48; CHECK-LABEL: ld1rqb_i8_imm_upper_bound:49; CHECK: // %bb.0:50; CHECK-NEXT: ld1rqb { z0.b }, p0/z, [x0, #112]51; CHECK-NEXT: ret52 %ptr = getelementptr inbounds i8, ptr %addr, i8 11253 %res = call <vscale x 16 x i8> @llvm.aarch64.sve.ld1rq.nxv16i8(<vscale x 16 x i1> %pred, ptr %ptr)54 ret <vscale x 16 x i8> %res55}56 57define <vscale x 16 x i8> @ld1rqb_i8_imm_out_of_lower_bound(<vscale x 16 x i1> %pred, ptr %addr) {58; CHECK-LABEL: ld1rqb_i8_imm_out_of_lower_bound:59; CHECK: // %bb.0:60; CHECK-NEXT: mov x8, #-12961; CHECK-NEXT: ld1rqb { z0.b }, p0/z, [x0, x8]62; CHECK-NEXT: ret63 %ptr = getelementptr inbounds i8, ptr %addr, i64 -12964 %res = call <vscale x 16 x i8> @llvm.aarch64.sve.ld1rq.nxv16i8(<vscale x 16 x i1> %pred, ptr %ptr)65 ret <vscale x 16 x i8> %res66}67 68define <vscale x 16 x i8> @ld1rqb_i8_imm_out_of_upper_bound(<vscale x 16 x i1> %pred, ptr %addr) {69; CHECK-LABEL: ld1rqb_i8_imm_out_of_upper_bound:70; CHECK: // %bb.0:71; CHECK-NEXT: mov w8, #11372; CHECK-NEXT: ld1rqb { z0.b }, p0/z, [x0, x8]73; CHECK-NEXT: ret74 %ptr = getelementptr inbounds i8, ptr %addr, i64 11375 %res = call <vscale x 16 x i8> @llvm.aarch64.sve.ld1rq.nxv16i8(<vscale x 16 x i1> %pred, ptr %ptr)76 ret <vscale x 16 x i8> %res77}78 79define <vscale x 16 x i8> @ld1rqb_i8_imm_dupqlane(<vscale x 8 x i1> %pred, ptr %addr) {80; CHECK-LABEL: ld1rqb_i8_imm_dupqlane:81; CHECK: // %bb.0:82; CHECK-NEXT: ptrue p0.b83; CHECK-NEXT: ld1rqb { z0.b }, p0/z, [x0, #-16]84; CHECK-NEXT: ret85 %ptr = getelementptr inbounds <16 x i8>, ptr %addr, i16 -186 %load = load <16 x i8>, ptr %ptr87 %1 = tail call <vscale x 16 x i8> @llvm.vector.insert.nxv16i8.v16i8(<vscale x 16 x i8> poison, <16 x i8> %load, i64 0)88 %2 = tail call <vscale x 16 x i8> @llvm.aarch64.sve.dupq.lane.nxv16i8(<vscale x 16 x i8> %1, i64 0)89 ret <vscale x 16 x i8> %290}91 92define <vscale x 16 x i8> @ld1rqb_i8_scalar_dupqlane(<vscale x 8 x i1> %pred, ptr %addr, i64 %idx) {93; CHECK-LABEL: ld1rqb_i8_scalar_dupqlane:94; CHECK: // %bb.0:95; CHECK-NEXT: ptrue p0.b96; CHECK-NEXT: ld1rqb { z0.b }, p0/z, [x0, x1]97; CHECK-NEXT: ret98 %ptr = getelementptr inbounds i8, ptr %addr, i64 %idx99 %load = load <16 x i8>, ptr %ptr100 %1 = tail call <vscale x 16 x i8> @llvm.vector.insert.nxv16i8.v16i8(<vscale x 16 x i8> poison, <16 x i8> %load, i64 0)101 %2 = tail call <vscale x 16 x i8> @llvm.aarch64.sve.dupq.lane.nxv16i8(<vscale x 16 x i8> %1, i64 0)102 ret <vscale x 16 x i8> %2103}104 105;106; LD1RQH107;108 109define <vscale x 8 x i16> @ld1rqh_i16(<vscale x 8 x i1> %pred, ptr %addr) {110; CHECK-LABEL: ld1rqh_i16:111; CHECK: // %bb.0:112; CHECK-NEXT: ld1rqh { z0.h }, p0/z, [x0]113; CHECK-NEXT: ret114 %res = call <vscale x 8 x i16> @llvm.aarch64.sve.ld1rq.nxv8i16(<vscale x 8 x i1> %pred, ptr %addr)115 ret <vscale x 8 x i16> %res116}117 118define <vscale x 8 x half> @ld1rqh_f16(<vscale x 8 x i1> %pred, ptr %addr) {119; CHECK-LABEL: ld1rqh_f16:120; CHECK: // %bb.0:121; CHECK-NEXT: ld1rqh { z0.h }, p0/z, [x0]122; CHECK-NEXT: ret123 %res = call <vscale x 8 x half> @llvm.aarch64.sve.ld1rq.nxv8f16(<vscale x 8 x i1> %pred, ptr %addr)124 ret <vscale x 8 x half> %res125}126 127define <vscale x 8 x i16> @ld1rqh_i16_imm(<vscale x 8 x i1> %pred, ptr %addr) {128; CHECK-LABEL: ld1rqh_i16_imm:129; CHECK: // %bb.0:130; CHECK-NEXT: ld1rqh { z0.h }, p0/z, [x0, #-64]131; CHECK-NEXT: ret132 %ptr = getelementptr inbounds i16, ptr %addr, i16 -32133 %res = call <vscale x 8 x i16> @llvm.aarch64.sve.ld1rq.nxv8i16(<vscale x 8 x i1> %pred, ptr %ptr)134 ret <vscale x 8 x i16> %res135}136 137define <vscale x 8 x half> @ld1rqh_f16_imm(<vscale x 8 x i1> %pred, ptr %addr) {138; CHECK-LABEL: ld1rqh_f16_imm:139; CHECK: // %bb.0:140; CHECK-NEXT: ld1rqh { z0.h }, p0/z, [x0, #-16]141; CHECK-NEXT: ret142 %ptr = getelementptr inbounds half, ptr %addr, i16 -8143 %res = call <vscale x 8 x half> @llvm.aarch64.sve.ld1rq.nxv8f16(<vscale x 8 x i1> %pred, ptr %ptr)144 ret <vscale x 8 x half> %res145}146 147define <vscale x 8 x i16> @ld1rqh_i16_scalar(<vscale x 8 x i1> %pred, ptr %addr, i64 %idx) {148; CHECK-LABEL: ld1rqh_i16_scalar:149; CHECK: // %bb.0:150; CHECK-NEXT: ld1rqh { z0.h }, p0/z, [x0, x1, lsl #1]151; CHECK-NEXT: ret152 %ptr = getelementptr inbounds i16, ptr %addr, i64 %idx153 %res = call <vscale x 8 x i16> @llvm.aarch64.sve.ld1rq.nxv8i16(<vscale x 8 x i1> %pred, ptr %ptr)154 ret <vscale x 8 x i16> %res155}156 157define <vscale x 8 x half> @ld1rqh_f16_scalar(<vscale x 8 x i1> %pred, ptr %addr, i64 %idx) {158; CHECK-LABEL: ld1rqh_f16_scalar:159; CHECK: // %bb.0:160; CHECK-NEXT: ld1rqh { z0.h }, p0/z, [x0, x1, lsl #1]161; CHECK-NEXT: ret162 %ptr = getelementptr inbounds half, ptr %addr, i64 %idx163 %res = call <vscale x 8 x half> @llvm.aarch64.sve.ld1rq.nxv8f16(<vscale x 8 x i1> %pred, ptr %ptr)164 ret <vscale x 8 x half> %res165}166 167define <vscale x 8 x bfloat> @ld1rqh_bf16(<vscale x 8 x i1> %pred, ptr %addr) {168; CHECK-LABEL: ld1rqh_bf16:169; CHECK: // %bb.0:170; CHECK-NEXT: ld1rqh { z0.h }, p0/z, [x0]171; CHECK-NEXT: ret172 %res = call <vscale x 8 x bfloat> @llvm.aarch64.sve.ld1rq.nxv8bf16(<vscale x 8 x i1> %pred, ptr %addr)173 ret <vscale x 8 x bfloat> %res174}175 176define <vscale x 8 x bfloat> @ld1rqh_bf16_imm(<vscale x 8 x i1> %pred, ptr %addr) {177; CHECK-LABEL: ld1rqh_bf16_imm:178; CHECK: // %bb.0:179; CHECK-NEXT: ld1rqh { z0.h }, p0/z, [x0, #-16]180; CHECK-NEXT: ret181 %ptr = getelementptr inbounds bfloat, ptr %addr, i16 -8182 %res = call <vscale x 8 x bfloat> @llvm.aarch64.sve.ld1rq.nxv8bf16(<vscale x 8 x i1> %pred, ptr %ptr)183 ret <vscale x 8 x bfloat> %res184}185 186define <vscale x 8 x bfloat> @ld1rqh_bf16_scalar(<vscale x 8 x i1> %pred, ptr %addr, i64 %idx) {187; CHECK-LABEL: ld1rqh_bf16_scalar:188; CHECK: // %bb.0:189; CHECK-NEXT: ld1rqh { z0.h }, p0/z, [x0, x1, lsl #1]190; CHECK-NEXT: ret191 %ptr = getelementptr inbounds bfloat, ptr %addr, i64 %idx192 %res = call <vscale x 8 x bfloat> @llvm.aarch64.sve.ld1rq.nxv8bf16(<vscale x 8 x i1> %pred, ptr %ptr)193 ret <vscale x 8 x bfloat> %res194}195 196define <vscale x 8 x i16> @ld1rqh_i16_imm_dupqlane(<vscale x 8 x i1> %pred, ptr %addr) {197; CHECK-LABEL: ld1rqh_i16_imm_dupqlane:198; CHECK: // %bb.0:199; CHECK-NEXT: ptrue p0.h200; CHECK-NEXT: ld1rqh { z0.h }, p0/z, [x0, #-16]201; CHECK-NEXT: ret202 %ptr = getelementptr inbounds <8 x i16>, ptr %addr, i16 -1203 %load = load <8 x i16>, ptr %ptr204 %1 = tail call <vscale x 8 x i16> @llvm.vector.insert.nxv8i16.v8i16(<vscale x 8 x i16> poison, <8 x i16> %load, i64 0)205 %2 = tail call <vscale x 8 x i16> @llvm.aarch64.sve.dupq.lane.nxv8i16(<vscale x 8 x i16> %1, i64 0)206 ret <vscale x 8 x i16> %2207}208 209define <vscale x 8 x i16> @ld1rqh_i16_scalar_dupqlane(<vscale x 8 x i1> %pred, ptr %addr, i64 %idx) {210; CHECK-LABEL: ld1rqh_i16_scalar_dupqlane:211; CHECK: // %bb.0:212; CHECK-NEXT: ptrue p0.h213; CHECK-NEXT: ld1rqh { z0.h }, p0/z, [x0, x1, lsl #1]214; CHECK-NEXT: ret215 %ptr = getelementptr inbounds i16, ptr %addr, i64 %idx216 %load = load <8 x i16>, ptr %ptr217 %1 = tail call <vscale x 8 x i16> @llvm.vector.insert.nxv8i16.v8i16(<vscale x 8 x i16> poison, <8 x i16> %load, i64 0)218 %2 = tail call <vscale x 8 x i16> @llvm.aarch64.sve.dupq.lane.nxv8i16(<vscale x 8 x i16> %1, i64 0)219 ret <vscale x 8 x i16> %2220}221 222define <vscale x 8 x half> @ld1rqh_f16_imm_dupqlane(<vscale x 8 x i1> %pred, ptr %addr) {223; CHECK-LABEL: ld1rqh_f16_imm_dupqlane:224; CHECK: // %bb.0:225; CHECK-NEXT: ptrue p0.h226; CHECK-NEXT: ld1rqh { z0.h }, p0/z, [x0, #-16]227; CHECK-NEXT: ret228 %ptr = getelementptr inbounds <8 x half>, ptr %addr, i16 -1229 %load = load <8 x half>, ptr %ptr230 %1 = tail call <vscale x 8 x half> @llvm.vector.insert.nxv8f16.v8f16(<vscale x 8 x half> poison, <8 x half> %load, i64 0)231 %2 = tail call <vscale x 8 x half> @llvm.aarch64.sve.dupq.lane.nxv8f16(<vscale x 8 x half> %1, i64 0)232 ret <vscale x 8 x half> %2233}234 235define <vscale x 8 x half> @ld1rqh_f16_scalar_dupqlane(<vscale x 8 x i1> %pred, ptr %addr, i64 %idx) {236; CHECK-LABEL: ld1rqh_f16_scalar_dupqlane:237; CHECK: // %bb.0:238; CHECK-NEXT: ptrue p0.h239; CHECK-NEXT: ld1rqh { z0.h }, p0/z, [x0, x1, lsl #1]240; CHECK-NEXT: ret241 %ptr = getelementptr inbounds half, ptr %addr, i64 %idx242 %load = load <8 x half>, ptr %ptr243 %1 = tail call <vscale x 8 x half> @llvm.vector.insert.nxv8f16.v8f16(<vscale x 8 x half> poison, <8 x half> %load, i64 0)244 %2 = tail call <vscale x 8 x half> @llvm.aarch64.sve.dupq.lane.nxv8f16(<vscale x 8 x half> %1, i64 0)245 ret <vscale x 8 x half> %2246}247 248define <vscale x 8 x bfloat> @ld1rqh_bf16_imm_dupqlane(<vscale x 8 x i1> %pred, ptr %addr) {249; CHECK-LABEL: ld1rqh_bf16_imm_dupqlane:250; CHECK: // %bb.0:251; CHECK-NEXT: ptrue p0.h252; CHECK-NEXT: ld1rqh { z0.h }, p0/z, [x0, #-16]253; CHECK-NEXT: ret254 %ptr = getelementptr inbounds <8 x bfloat>, ptr %addr, i16 -1255 %load = load <8 x bfloat>, ptr %ptr256 %1 = tail call <vscale x 8 x bfloat> @llvm.vector.insert.nxv8bf16.v8bf16(<vscale x 8 x bfloat> poison, <8 x bfloat> %load, i64 0)257 %2 = tail call <vscale x 8 x bfloat> @llvm.aarch64.sve.dupq.lane.nxv8bf16(<vscale x 8 x bfloat> %1, i64 0)258 ret <vscale x 8 x bfloat> %2259}260 261define <vscale x 8 x bfloat> @ld1rqh_bf16_scalar_dupqlane(<vscale x 8 x i1> %pred, ptr %addr, i64 %idx) {262; CHECK-LABEL: ld1rqh_bf16_scalar_dupqlane:263; CHECK: // %bb.0:264; CHECK-NEXT: ptrue p0.h265; CHECK-NEXT: ld1rqh { z0.h }, p0/z, [x0, x1, lsl #1]266; CHECK-NEXT: ret267 %ptr = getelementptr inbounds bfloat, ptr %addr, i64 %idx268 %load = load <8 x bfloat>, ptr %ptr269 %1 = tail call <vscale x 8 x bfloat> @llvm.vector.insert.nxv8bf16.v8bf16(<vscale x 8 x bfloat> poison, <8 x bfloat> %load, i64 0)270 %2 = tail call <vscale x 8 x bfloat> @llvm.aarch64.sve.dupq.lane.nxv8bf16(<vscale x 8 x bfloat> %1, i64 0)271 ret <vscale x 8 x bfloat> %2272}273 274;275; LD1RQW276;277 278define <vscale x 4 x i32> @ld1rqw_i32(<vscale x 4 x i1> %pred, ptr %addr) {279; CHECK-LABEL: ld1rqw_i32:280; CHECK: // %bb.0:281; CHECK-NEXT: ld1rqw { z0.s }, p0/z, [x0]282; CHECK-NEXT: ret283 %res = call <vscale x 4 x i32> @llvm.aarch64.sve.ld1rq.nxv4i32(<vscale x 4 x i1> %pred, ptr %addr)284 ret <vscale x 4 x i32> %res285}286 287define <vscale x 4 x float> @ld1rqw_f32(<vscale x 4 x i1> %pred, ptr %addr) {288; CHECK-LABEL: ld1rqw_f32:289; CHECK: // %bb.0:290; CHECK-NEXT: ld1rqw { z0.s }, p0/z, [x0]291; CHECK-NEXT: ret292 %res = call <vscale x 4 x float> @llvm.aarch64.sve.ld1rq.nxv4f32(<vscale x 4 x i1> %pred, ptr %addr)293 ret <vscale x 4 x float> %res294}295 296define <vscale x 4 x i32> @ld1rqw_i32_imm(<vscale x 4 x i1> %pred, ptr %addr) {297; CHECK-LABEL: ld1rqw_i32_imm:298; CHECK: // %bb.0:299; CHECK-NEXT: ld1rqw { z0.s }, p0/z, [x0, #112]300; CHECK-NEXT: ret301 %ptr = getelementptr inbounds i32, ptr %addr, i32 28302 %res = call <vscale x 4 x i32> @llvm.aarch64.sve.ld1rq.nxv4i32(<vscale x 4 x i1> %pred, ptr %ptr)303 ret <vscale x 4 x i32> %res304}305 306define <vscale x 4 x float> @ld1rqw_f32_imm(<vscale x 4 x i1> %pred, ptr %addr) {307; CHECK-LABEL: ld1rqw_f32_imm:308; CHECK: // %bb.0:309; CHECK-NEXT: ld1rqw { z0.s }, p0/z, [x0, #32]310; CHECK-NEXT: ret311 %ptr = getelementptr inbounds float, ptr %addr, i32 8312 %res = call <vscale x 4 x float> @llvm.aarch64.sve.ld1rq.nxv4f32(<vscale x 4 x i1> %pred, ptr %ptr)313 ret <vscale x 4 x float> %res314}315 316define <vscale x 4 x i32> @ld1rqw_i32_scalar(<vscale x 4 x i1> %pred, ptr %base, i64 %idx) {317; CHECK-LABEL: ld1rqw_i32_scalar:318; CHECK: // %bb.0:319; CHECK-NEXT: ld1rqw { z0.s }, p0/z, [x0, x1, lsl #2]320; CHECK-NEXT: ret321 %ptr = getelementptr inbounds i32, ptr %base, i64 %idx322 %res = call <vscale x 4 x i32> @llvm.aarch64.sve.ld1rq.nxv4i32(<vscale x 4 x i1> %pred, ptr %ptr)323 ret <vscale x 4 x i32> %res324}325 326define <vscale x 4 x float> @ld1rqw_f32_scalar(<vscale x 4 x i1> %pred, ptr %base, i64 %idx) {327; CHECK-LABEL: ld1rqw_f32_scalar:328; CHECK: // %bb.0:329; CHECK-NEXT: ld1rqw { z0.s }, p0/z, [x0, x1, lsl #2]330; CHECK-NEXT: ret331 %ptr = getelementptr inbounds float, ptr %base, i64 %idx332 %res = call <vscale x 4 x float> @llvm.aarch64.sve.ld1rq.nxv4f32(<vscale x 4 x i1> %pred, ptr %ptr)333 ret <vscale x 4 x float> %res334}335 336define <vscale x 4 x i32> @ld1rqw_i32_imm_dupqlane(<vscale x 4 x i1> %pred, ptr %addr) {337; CHECK-LABEL: ld1rqw_i32_imm_dupqlane:338; CHECK: // %bb.0:339; CHECK-NEXT: ptrue p0.s340; CHECK-NEXT: ld1rqw { z0.s }, p0/z, [x0, #16]341; CHECK-NEXT: ret342 %ptr = getelementptr inbounds <4 x i32>, ptr %addr, i32 1343 %load = load <4 x i32>, ptr %ptr344 %1 = tail call <vscale x 4 x i32> @llvm.vector.insert.nxv4i32.v4i32(<vscale x 4 x i32> poison, <4 x i32> %load, i64 0)345 %2 = tail call <vscale x 4 x i32> @llvm.aarch64.sve.dupq.lane.nxv4i32(<vscale x 4 x i32> %1, i64 0)346 ret <vscale x 4 x i32> %2347}348 349define <vscale x 4 x i32> @ld1rqw_i32_scalar_dupqlane(<vscale x 4 x i1> %pred, ptr %addr, i64 %idx) {350; CHECK-LABEL: ld1rqw_i32_scalar_dupqlane:351; CHECK: // %bb.0:352; CHECK-NEXT: ptrue p0.s353; CHECK-NEXT: ld1rqw { z0.s }, p0/z, [x0, x1, lsl #2]354; CHECK-NEXT: ret355 %ptr = getelementptr inbounds i32, ptr %addr, i64 %idx356 %load = load <4 x i32>, ptr %ptr357 %1 = tail call <vscale x 4 x i32> @llvm.vector.insert.nxv4i32.v4i32(<vscale x 4 x i32> poison, <4 x i32> %load, i64 0)358 %2 = tail call <vscale x 4 x i32> @llvm.aarch64.sve.dupq.lane.nxv4i32(<vscale x 4 x i32> %1, i64 0)359 ret <vscale x 4 x i32> %2360}361 362define <vscale x 4 x float> @ld1rqw_f32_imm_dupqlane(<vscale x 4 x i1> %pred, ptr %addr) {363; CHECK-LABEL: ld1rqw_f32_imm_dupqlane:364; CHECK: // %bb.0:365; CHECK-NEXT: ptrue p0.s366; CHECK-NEXT: ld1rqw { z0.s }, p0/z, [x0, #16]367; CHECK-NEXT: ret368 %ptr = getelementptr inbounds <4 x float>, ptr %addr, i32 1369 %load = load <4 x float>, ptr %ptr370 %1 = tail call <vscale x 4 x float> @llvm.vector.insert.nxv4f32.v4f32(<vscale x 4 x float> poison, <4 x float> %load, i64 0)371 %2 = tail call <vscale x 4 x float> @llvm.aarch64.sve.dupq.lane.nxv4f32(<vscale x 4 x float> %1, i64 0)372 ret <vscale x 4 x float> %2373}374 375define <vscale x 4 x float> @ld1rqw_f32_scalar_dupqlane(<vscale x 4 x i1> %pred, ptr %addr, i64 %idx) {376; CHECK-LABEL: ld1rqw_f32_scalar_dupqlane:377; CHECK: // %bb.0:378; CHECK-NEXT: ptrue p0.s379; CHECK-NEXT: ld1rqw { z0.s }, p0/z, [x0, x1, lsl #2]380; CHECK-NEXT: ret381 %ptr = getelementptr inbounds float, ptr %addr, i64 %idx382 %load = load <4 x float>, ptr %ptr383 %1 = tail call <vscale x 4 x float> @llvm.vector.insert.nxv4f32.v4f32(<vscale x 4 x float> poison, <4 x float> %load, i64 0)384 %2 = tail call <vscale x 4 x float> @llvm.aarch64.sve.dupq.lane.nxv4f32(<vscale x 4 x float> %1, i64 0)385 ret <vscale x 4 x float> %2386}387 388;389; LD1RQD390;391 392define <vscale x 2 x i64> @ld1rqd_i64(<vscale x 2 x i1> %pred, ptr %addr) {393; CHECK-LABEL: ld1rqd_i64:394; CHECK: // %bb.0:395; CHECK-NEXT: ld1rqd { z0.d }, p0/z, [x0]396; CHECK-NEXT: ret397 %res = call <vscale x 2 x i64> @llvm.aarch64.sve.ld1rq.nxv2i64(<vscale x 2 x i1> %pred, ptr %addr)398 ret <vscale x 2 x i64> %res399}400 401define <vscale x 2 x double> @ld1rqd_f64(<vscale x 2 x i1> %pred, ptr %addr) {402; CHECK-LABEL: ld1rqd_f64:403; CHECK: // %bb.0:404; CHECK-NEXT: ld1rqd { z0.d }, p0/z, [x0]405; CHECK-NEXT: ret406 %res = call <vscale x 2 x double> @llvm.aarch64.sve.ld1rq.nxv2f64(<vscale x 2 x i1> %pred, ptr %addr)407 ret <vscale x 2 x double> %res408}409 410define <vscale x 2 x i64> @ld1rqd_i64_imm(<vscale x 2 x i1> %pred, ptr %addr) {411; CHECK-LABEL: ld1rqd_i64_imm:412; CHECK: // %bb.0:413; CHECK-NEXT: ld1rqd { z0.d }, p0/z, [x0, #64]414; CHECK-NEXT: ret415 %ptr = getelementptr inbounds i64, ptr %addr, i64 8416 %res = call <vscale x 2 x i64> @llvm.aarch64.sve.ld1rq.nxv2i64(<vscale x 2 x i1> %pred, ptr %ptr)417 ret <vscale x 2 x i64> %res418}419 420define <vscale x 2 x double> @ld1rqd_f64_imm(<vscale x 2 x i1> %pred, ptr %addr) {421; CHECK-LABEL: ld1rqd_f64_imm:422; CHECK: // %bb.0:423; CHECK-NEXT: ld1rqd { z0.d }, p0/z, [x0, #-128]424; CHECK-NEXT: ret425 %ptr = getelementptr inbounds double, ptr %addr, i64 -16426 %res = call <vscale x 2 x double> @llvm.aarch64.sve.ld1rq.nxv2f64(<vscale x 2 x i1> %pred, ptr %ptr)427 ret <vscale x 2 x double> %res428}429 430define <vscale x 2 x i64> @ld1rqd_i64_scalar(<vscale x 2 x i1> %pred, ptr %base, i64 %idx) {431; CHECK-LABEL: ld1rqd_i64_scalar:432; CHECK: // %bb.0:433; CHECK-NEXT: ld1rqd { z0.d }, p0/z, [x0, x1, lsl #3]434; CHECK-NEXT: ret435 %ptr = getelementptr inbounds i64, ptr %base, i64 %idx436 %res = call <vscale x 2 x i64> @llvm.aarch64.sve.ld1rq.nxv2i64(<vscale x 2 x i1> %pred, ptr %ptr)437 ret <vscale x 2 x i64> %res438}439 440define <vscale x 2 x double> @ld1rqd_f64_scalar(<vscale x 2 x i1> %pred, ptr %base, i64 %idx) {441; CHECK-LABEL: ld1rqd_f64_scalar:442; CHECK: // %bb.0:443; CHECK-NEXT: ld1rqd { z0.d }, p0/z, [x0, x1, lsl #3]444; CHECK-NEXT: ret445 %ptr = getelementptr inbounds double, ptr %base, i64 %idx446 %res = call <vscale x 2 x double> @llvm.aarch64.sve.ld1rq.nxv2f64(<vscale x 2 x i1> %pred, ptr %ptr)447 ret <vscale x 2 x double> %res448}449 450define <vscale x 2 x i64> @ld1rqd_i64_imm_dupqlane(<vscale x 2 x i1> %pred, ptr %addr) {451; CHECK-LABEL: ld1rqd_i64_imm_dupqlane:452; CHECK: // %bb.0:453; CHECK-NEXT: ptrue p0.d454; CHECK-NEXT: ld1rqd { z0.d }, p0/z, [x0, #16]455; CHECK-NEXT: ret456 %ptr = getelementptr inbounds <2 x i64>, ptr %addr, i64 1457 %load = load <2 x i64>, ptr %ptr458 %1 = tail call <vscale x 2 x i64> @llvm.vector.insert.nxv2i64.v2i64(<vscale x 2 x i64> poison, <2 x i64> %load, i64 0)459 %2 = tail call <vscale x 2 x i64> @llvm.aarch64.sve.dupq.lane.nxv2i64(<vscale x 2 x i64> %1, i64 0)460 ret <vscale x 2 x i64> %2461}462 463define <vscale x 2 x i64> @ld1rqd_i64_scalar_dupqlane(<vscale x 2 x i1> %pred, ptr %addr, i64 %idx) {464; CHECK-LABEL: ld1rqd_i64_scalar_dupqlane:465; CHECK: // %bb.0:466; CHECK-NEXT: ptrue p0.d467; CHECK-NEXT: ld1rqd { z0.d }, p0/z, [x0, x1, lsl #3]468; CHECK-NEXT: ret469 %ptr = getelementptr inbounds i64, ptr %addr, i64 %idx470 %load = load <2 x i64>, ptr %ptr471 %1 = tail call <vscale x 2 x i64> @llvm.vector.insert.nxv2i64.v2i64(<vscale x 2 x i64> poison, <2 x i64> %load, i64 0)472 %2 = tail call <vscale x 2 x i64> @llvm.aarch64.sve.dupq.lane.nxv2i64(<vscale x 2 x i64> %1, i64 0)473 ret <vscale x 2 x i64> %2474}475 476define <vscale x 2 x double> @ld1rqd_f64_imm_dupqlane(<vscale x 2 x i1> %pred, ptr %addr) {477; CHECK-LABEL: ld1rqd_f64_imm_dupqlane:478; CHECK: // %bb.0:479; CHECK-NEXT: ptrue p0.d480; CHECK-NEXT: ld1rqd { z0.d }, p0/z, [x0, #16]481; CHECK-NEXT: ret482 %ptr = getelementptr inbounds <2 x double>, ptr %addr, i64 1483 %load = load <2 x double>, ptr %ptr484 %1 = tail call <vscale x 2 x double> @llvm.vector.insert.nxv2f64.v2f64(<vscale x 2 x double> poison, <2 x double> %load, i64 0)485 %2 = tail call <vscale x 2 x double> @llvm.aarch64.sve.dupq.lane.nxv2f64(<vscale x 2 x double> %1, i64 0)486 ret <vscale x 2 x double> %2487}488 489define <vscale x 2 x double> @ld1rqd_f64_scalar_dupqlane(<vscale x 2 x i1> %pred, ptr %addr, i64 %idx) {490; CHECK-LABEL: ld1rqd_f64_scalar_dupqlane:491; CHECK: // %bb.0:492; CHECK-NEXT: ptrue p0.d493; CHECK-NEXT: ld1rqd { z0.d }, p0/z, [x0, x1, lsl #3]494; CHECK-NEXT: ret495 %ptr = getelementptr inbounds double, ptr %addr, i64 %idx496 %load = load <2 x double>, ptr %ptr497 %1 = tail call <vscale x 2 x double> @llvm.vector.insert.nxv2f64.v2f64(<vscale x 2 x double> poison, <2 x double> %load, i64 0)498 %2 = tail call <vscale x 2 x double> @llvm.aarch64.sve.dupq.lane.nxv2f64(<vscale x 2 x double> %1, i64 0)499 ret <vscale x 2 x double> %2500}501 502;503; LDNT1B504;505 506define <vscale x 16 x i8> @ldnt1b_i8(<vscale x 16 x i1> %pred, ptr %addr) {507; CHECK-LABEL: ldnt1b_i8:508; CHECK: // %bb.0:509; CHECK-NEXT: ldnt1b { z0.b }, p0/z, [x0]510; CHECK-NEXT: ret511 %res = call <vscale x 16 x i8> @llvm.aarch64.sve.ldnt1.nxv16i8(<vscale x 16 x i1> %pred,512 ptr %addr)513 ret <vscale x 16 x i8> %res514}515 516;517; LDNT1H518;519 520define <vscale x 8 x i16> @ldnt1h_i16(<vscale x 8 x i1> %pred, ptr %addr) {521; CHECK-LABEL: ldnt1h_i16:522; CHECK: // %bb.0:523; CHECK-NEXT: ldnt1h { z0.h }, p0/z, [x0]524; CHECK-NEXT: ret525 %res = call <vscale x 8 x i16> @llvm.aarch64.sve.ldnt1.nxv8i16(<vscale x 8 x i1> %pred,526 ptr %addr)527 ret <vscale x 8 x i16> %res528}529 530define <vscale x 8 x half> @ldnt1h_f16(<vscale x 8 x i1> %pred, ptr %addr) {531; CHECK-LABEL: ldnt1h_f16:532; CHECK: // %bb.0:533; CHECK-NEXT: ldnt1h { z0.h }, p0/z, [x0]534; CHECK-NEXT: ret535 %res = call <vscale x 8 x half> @llvm.aarch64.sve.ldnt1.nxv8f16(<vscale x 8 x i1> %pred,536 ptr %addr)537 ret <vscale x 8 x half> %res538}539 540define <vscale x 8 x bfloat> @ldnt1h_bf16(<vscale x 8 x i1> %pred, ptr %addr) {541; CHECK-LABEL: ldnt1h_bf16:542; CHECK: // %bb.0:543; CHECK-NEXT: ldnt1h { z0.h }, p0/z, [x0]544; CHECK-NEXT: ret545 %res = call <vscale x 8 x bfloat> @llvm.aarch64.sve.ldnt1.nxv8bf16(<vscale x 8 x i1> %pred,546 ptr %addr)547 ret <vscale x 8 x bfloat> %res548}549 550;551; LDNT1W552;553 554define <vscale x 4 x i32> @ldnt1w_i32(<vscale x 4 x i1> %pred, ptr %addr) {555; CHECK-LABEL: ldnt1w_i32:556; CHECK: // %bb.0:557; CHECK-NEXT: ldnt1w { z0.s }, p0/z, [x0]558; CHECK-NEXT: ret559 %res = call <vscale x 4 x i32> @llvm.aarch64.sve.ldnt1.nxv4i32(<vscale x 4 x i1> %pred,560 ptr %addr)561 ret <vscale x 4 x i32> %res562}563 564define <vscale x 4 x float> @ldnt1w_f32(<vscale x 4 x i1> %pred, ptr %addr) {565; CHECK-LABEL: ldnt1w_f32:566; CHECK: // %bb.0:567; CHECK-NEXT: ldnt1w { z0.s }, p0/z, [x0]568; CHECK-NEXT: ret569 %res = call <vscale x 4 x float> @llvm.aarch64.sve.ldnt1.nxv4f32(<vscale x 4 x i1> %pred,570 ptr %addr)571 ret <vscale x 4 x float> %res572}573 574;575; LDNT1D576;577 578define <vscale x 2 x i64> @ldnt1d_i64(<vscale x 2 x i1> %pred, ptr %addr) {579; CHECK-LABEL: ldnt1d_i64:580; CHECK: // %bb.0:581; CHECK-NEXT: ldnt1d { z0.d }, p0/z, [x0]582; CHECK-NEXT: ret583 %res = call <vscale x 2 x i64> @llvm.aarch64.sve.ldnt1.nxv2i64(<vscale x 2 x i1> %pred,584 ptr %addr)585 ret <vscale x 2 x i64> %res586}587 588define <vscale x 2 x double> @ldnt1d_f64(<vscale x 2 x i1> %pred, ptr %addr) {589; CHECK-LABEL: ldnt1d_f64:590; CHECK: // %bb.0:591; CHECK-NEXT: ldnt1d { z0.d }, p0/z, [x0]592; CHECK-NEXT: ret593 %res = call <vscale x 2 x double> @llvm.aarch64.sve.ldnt1.nxv2f64(<vscale x 2 x i1> %pred,594 ptr %addr)595 ret <vscale x 2 x double> %res596}597 598 599declare <vscale x 16 x i8> @llvm.aarch64.sve.ld1rq.nxv16i8(<vscale x 16 x i1>, ptr)600declare <vscale x 8 x i16> @llvm.aarch64.sve.ld1rq.nxv8i16(<vscale x 8 x i1>, ptr)601declare <vscale x 4 x i32> @llvm.aarch64.sve.ld1rq.nxv4i32(<vscale x 4 x i1>, ptr)602declare <vscale x 2 x i64> @llvm.aarch64.sve.ld1rq.nxv2i64(<vscale x 2 x i1>, ptr)603declare <vscale x 8 x half> @llvm.aarch64.sve.ld1rq.nxv8f16(<vscale x 8 x i1>, ptr)604declare <vscale x 8 x bfloat> @llvm.aarch64.sve.ld1rq.nxv8bf16(<vscale x 8 x i1>, ptr)605declare <vscale x 4 x float> @llvm.aarch64.sve.ld1rq.nxv4f32(<vscale x 4 x i1>, ptr)606declare <vscale x 2 x double> @llvm.aarch64.sve.ld1rq.nxv2f64(<vscale x 2 x i1>, ptr)607 608declare <vscale x 16 x i8> @llvm.aarch64.sve.ldnt1.nxv16i8(<vscale x 16 x i1>, ptr)609declare <vscale x 8 x i16> @llvm.aarch64.sve.ldnt1.nxv8i16(<vscale x 8 x i1>, ptr)610declare <vscale x 4 x i32> @llvm.aarch64.sve.ldnt1.nxv4i32(<vscale x 4 x i1>, ptr)611declare <vscale x 2 x i64> @llvm.aarch64.sve.ldnt1.nxv2i64(<vscale x 2 x i1>, ptr)612declare <vscale x 8 x half> @llvm.aarch64.sve.ldnt1.nxv8f16(<vscale x 8 x i1>, ptr)613declare <vscale x 8 x bfloat> @llvm.aarch64.sve.ldnt1.nxv8bf16(<vscale x 8 x i1>, ptr)614declare <vscale x 4 x float> @llvm.aarch64.sve.ldnt1.nxv4f32(<vscale x 4 x i1>, ptr)615declare <vscale x 2 x double> @llvm.aarch64.sve.ldnt1.nxv2f64(<vscale x 2 x i1>, ptr)616 617declare <vscale x 2 x i64> @llvm.vector.insert.nxv2i64.v2i64(<vscale x 2 x i64>, <2 x i64>, i64)618declare <vscale x 2 x double> @llvm.vector.insert.nxv2f64.v2f64(<vscale x 2 x double>, <2 x double>, i64)619declare <vscale x 4 x i32> @llvm.vector.insert.nxv4i32.v4i32(<vscale x 4 x i32>, <4 x i32>, i64)620declare <vscale x 4 x float> @llvm.vector.insert.nxv4f32.v4f32(<vscale x 4 x float>, <4 x float>, i64)621declare <vscale x 8 x i16> @llvm.vector.insert.nxv8i16.v8i16(<vscale x 8 x i16>, <8 x i16>, i64)622declare <vscale x 8 x half> @llvm.vector.insert.nxv8f16.v8f16(<vscale x 8 x half>, <8 x half>, i64)623declare <vscale x 8 x bfloat> @llvm.vector.insert.nxv8bf16.v8bf16(<vscale x 8 x bfloat>, <8 x bfloat>, i64)624declare <vscale x 16 x i8> @llvm.vector.insert.nxv16i8.v16i8(<vscale x 16 x i8>, <16 x i8>, i64)625 626declare <vscale x 2 x i64> @llvm.aarch64.sve.dupq.lane.nxv2i64(<vscale x 2 x i64>, i64)627declare <vscale x 2 x double> @llvm.aarch64.sve.dupq.lane.nxv2f64(<vscale x 2 x double>, i64)628declare <vscale x 4 x i32> @llvm.aarch64.sve.dupq.lane.nxv4i32(<vscale x 4 x i32>, i64)629declare <vscale x 4 x float> @llvm.aarch64.sve.dupq.lane.nxv4f32(<vscale x 4 x float>, i64)630declare <vscale x 8 x i16> @llvm.aarch64.sve.dupq.lane.nxv8i16(<vscale x 8 x i16>, i64)631declare <vscale x 8 x half> @llvm.aarch64.sve.dupq.lane.nxv8f16(<vscale x 8 x half>, i64)632declare <vscale x 8 x bfloat> @llvm.aarch64.sve.dupq.lane.nxv8bf16(<vscale x 8 x bfloat>, i64)633declare <vscale x 16 x i8> @llvm.aarch64.sve.dupq.lane.nxv16i8(<vscale x 16 x i8>, i64)634