540 lines · plain
1; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py2; RUN: llc -mtriple=aarch64--linux-gnu -mattr=sve < %s | FileCheck %s3; RUN: llc -mtriple=aarch64--linux-gnu -mattr=sme -force-streaming < %s | FileCheck %s4 5; NOTE: invalid, upper and lower bound immediate values of the regimm6; addressing mode are checked only for the byte version of each7; instruction (`ld<N>b`), as the code for detecting the immediate is8; common to all instructions, and varies only for the number of9; elements of the structure store, which is <N> = 2, 3, 4.10 11; ld2b12define { <vscale x 16 x i8>, <vscale x 16 x i8> } @ld2.nxv32i8(<vscale x 16 x i1> %Pg, ptr %addr) {13; CHECK-LABEL: ld2.nxv32i8:14; CHECK: // %bb.0:15; CHECK-NEXT: ld2b { z0.b, z1.b }, p0/z, [x0, #2, mul vl]16; CHECK-NEXT: ret17 %base = getelementptr <vscale x 16 x i8>, ptr %addr, i64 218 %base_ptr = bitcast ptr %base to ptr19 %res = call { <vscale x 16 x i8>, <vscale x 16 x i8> } @llvm.aarch64.sve.ld2.sret.nxv16i8(<vscale x 16 x i1> %Pg, ptr %base_ptr)20 ret { <vscale x 16 x i8>, <vscale x 16 x i8> } %res21}22 23define { <vscale x 16 x i8>, <vscale x 16 x i8> } @ld2.nxv32i8_lower_bound(<vscale x 16 x i1> %Pg, ptr %addr) {24; CHECK-LABEL: ld2.nxv32i8_lower_bound:25; CHECK: // %bb.0:26; CHECK-NEXT: ld2b { z0.b, z1.b }, p0/z, [x0, #-16, mul vl]27; CHECK-NEXT: ret28 %base = getelementptr <vscale x 16 x i8>, ptr %addr, i64 -1629 %base_ptr = bitcast ptr %base to ptr30 %res = call { <vscale x 16 x i8>, <vscale x 16 x i8> } @llvm.aarch64.sve.ld2.sret.nxv16i8(<vscale x 16 x i1> %Pg, ptr %base_ptr)31 ret { <vscale x 16 x i8>, <vscale x 16 x i8> } %res32}33 34define { <vscale x 16 x i8>, <vscale x 16 x i8> } @ld2.nxv32i8_upper_bound(<vscale x 16 x i1> %Pg, ptr %addr) {35; CHECK-LABEL: ld2.nxv32i8_upper_bound:36; CHECK: // %bb.0:37; CHECK-NEXT: ld2b { z0.b, z1.b }, p0/z, [x0, #14, mul vl]38; CHECK-NEXT: ret39 %base = getelementptr <vscale x 16 x i8>, ptr %addr, i64 1440 %base_ptr = bitcast ptr %base to ptr41 %res = call { <vscale x 16 x i8>, <vscale x 16 x i8> } @llvm.aarch64.sve.ld2.sret.nxv16i8(<vscale x 16 x i1> %Pg, ptr %base_ptr)42 ret { <vscale x 16 x i8>, <vscale x 16 x i8> } %res43}44 45define { <vscale x 16 x i8>, <vscale x 16 x i8> } @ld2.nxv32i8_not_multiple_of_2(<vscale x 16 x i1> %Pg, ptr %addr) {46; CHECK-LABEL: ld2.nxv32i8_not_multiple_of_2:47; CHECK: // %bb.0:48; CHECK-NEXT: rdvl x8, #349; CHECK-NEXT: ld2b { z0.b, z1.b }, p0/z, [x0, x8]50; CHECK-NEXT: ret51 %base = getelementptr <vscale x 16 x i8>, ptr %addr, i64 352 %base_ptr = bitcast ptr %base to ptr53 %res = call { <vscale x 16 x i8>, <vscale x 16 x i8> } @llvm.aarch64.sve.ld2.sret.nxv16i8(<vscale x 16 x i1> %Pg, ptr %base_ptr)54 ret { <vscale x 16 x i8>, <vscale x 16 x i8> } %res55}56 57define { <vscale x 16 x i8>, <vscale x 16 x i8> } @ld2.nxv32i8_outside_lower_bound(<vscale x 16 x i1> %Pg, ptr %addr) {58; CHECK-LABEL: ld2.nxv32i8_outside_lower_bound:59; CHECK: // %bb.0:60; CHECK-NEXT: rdvl x8, #-1861; CHECK-NEXT: ld2b { z0.b, z1.b }, p0/z, [x0, x8]62; CHECK-NEXT: ret63 %base = getelementptr <vscale x 16 x i8>, ptr %addr, i64 -1864 %base_ptr = bitcast ptr %base to ptr65 %res = call { <vscale x 16 x i8>, <vscale x 16 x i8> } @llvm.aarch64.sve.ld2.sret.nxv16i8(<vscale x 16 x i1> %Pg, ptr %base_ptr)66 ret { <vscale x 16 x i8>, <vscale x 16 x i8> } %res67}68 69define { <vscale x 16 x i8>, <vscale x 16 x i8> } @ld2.nxv32i8_outside_upper_bound(<vscale x 16 x i1> %Pg, ptr %addr) {70; CHECK-LABEL: ld2.nxv32i8_outside_upper_bound:71; CHECK: // %bb.0:72; CHECK-NEXT: rdvl x8, #1673; CHECK-NEXT: ld2b { z0.b, z1.b }, p0/z, [x0, x8]74; CHECK-NEXT: ret75 %base = getelementptr <vscale x 16 x i8>, ptr %addr, i64 1676 %base_ptr = bitcast ptr %base to ptr77 %res = call { <vscale x 16 x i8>, <vscale x 16 x i8> } @llvm.aarch64.sve.ld2.sret.nxv16i8(<vscale x 16 x i1> %Pg, ptr %base_ptr)78 ret { <vscale x 16 x i8>, <vscale x 16 x i8> } %res79}80 81; ld2h82define { <vscale x 8 x i16>, <vscale x 8 x i16> } @ld2.nxv16i16(<vscale x 8 x i1> %Pg, ptr %addr) {83; CHECK-LABEL: ld2.nxv16i16:84; CHECK: // %bb.0:85; CHECK-NEXT: ld2h { z0.h, z1.h }, p0/z, [x0, #14, mul vl]86; CHECK-NEXT: ret87 %base = getelementptr <vscale x 8 x i16>, ptr %addr, i64 1488 %base_ptr = bitcast ptr %base to ptr89 %res = call { <vscale x 8 x i16>, <vscale x 8 x i16> } @llvm.aarch64.sve.ld2.sret.nxv8i16(<vscale x 8 x i1> %Pg, ptr %base_ptr)90 ret { <vscale x 8 x i16>, <vscale x 8 x i16> } %res91}92 93define { <vscale x 8 x half>, <vscale x 8 x half> } @ld2.nxv16f16(<vscale x 8 x i1> %Pg, ptr %addr) {94; CHECK-LABEL: ld2.nxv16f16:95; CHECK: // %bb.0:96; CHECK-NEXT: ld2h { z0.h, z1.h }, p0/z, [x0, #-16, mul vl]97; CHECK-NEXT: ret98 %base = getelementptr <vscale x 8 x half>, ptr %addr, i64 -1699 %base_ptr = bitcast ptr %base to ptr100 %res = call { <vscale x 8 x half>, <vscale x 8 x half> } @llvm.aarch64.sve.ld2.sret.nxv8f16(<vscale x 8 x i1> %Pg, ptr %base_ptr)101 ret { <vscale x 8 x half>, <vscale x 8 x half> } %res102}103 104define { <vscale x 8 x bfloat>, <vscale x 8 x bfloat> } @ld2.nxv16bf16(<vscale x 8 x i1> %Pg, ptr %addr) #0 {105; CHECK-LABEL: ld2.nxv16bf16:106; CHECK: // %bb.0:107; CHECK-NEXT: ld2h { z0.h, z1.h }, p0/z, [x0, #12, mul vl]108; CHECK-NEXT: ret109 %base = getelementptr <vscale x 8 x bfloat>, ptr %addr, i64 12110 %base_ptr = bitcast ptr %base to ptr111 %res = call { <vscale x 8 x bfloat>, <vscale x 8 x bfloat> } @llvm.aarch64.sve.ld2.sret.nxv8bf16(<vscale x 8 x i1> %Pg, ptr %base_ptr)112 ret { <vscale x 8 x bfloat>, <vscale x 8 x bfloat> } %res113}114 115; ld2w116define { <vscale x 4 x i32>, <vscale x 4 x i32> } @ld2.nxv8i32(<vscale x 4 x i1> %Pg, ptr %addr) {117; CHECK-LABEL: ld2.nxv8i32:118; CHECK: // %bb.0:119; CHECK-NEXT: ld2w { z0.s, z1.s }, p0/z, [x0, #14, mul vl]120; CHECK-NEXT: ret121 %base = getelementptr <vscale x 4 x i32>, ptr %addr, i64 14122 %base_ptr = bitcast ptr %base to ptr123 %res = call { <vscale x 4 x i32>, <vscale x 4 x i32> } @llvm.aarch64.sve.ld2.sret.nxv4i32(<vscale x 4 x i1> %Pg, ptr %base_ptr)124 ret { <vscale x 4 x i32>, <vscale x 4 x i32> } %res125}126 127define { <vscale x 4 x float>, <vscale x 4 x float> } @ld2.nxv8f32(<vscale x 4 x i1> %Pg, ptr %addr) {128; CHECK-LABEL: ld2.nxv8f32:129; CHECK: // %bb.0:130; CHECK-NEXT: ld2w { z0.s, z1.s }, p0/z, [x0, #-16, mul vl]131; CHECK-NEXT: ret132 %base = getelementptr <vscale x 4 x float>, ptr %addr, i64 -16133 %base_ptr = bitcast ptr %base to ptr134 %res = call { <vscale x 4 x float>, <vscale x 4 x float> } @llvm.aarch64.sve.ld2.sret.nxv4f32(<vscale x 4 x i1> %Pg, ptr %base_ptr)135 ret { <vscale x 4 x float>, <vscale x 4 x float> } %res136}137 138; ld2d139define { <vscale x 2 x i64>, <vscale x 2 x i64> } @ld2.nxv4i64(<vscale x 2 x i1> %Pg, ptr %addr) {140; CHECK-LABEL: ld2.nxv4i64:141; CHECK: // %bb.0:142; CHECK-NEXT: ld2d { z0.d, z1.d }, p0/z, [x0, #14, mul vl]143; CHECK-NEXT: ret144 %base = getelementptr <vscale x 2 x i64>, ptr %addr, i64 14145 %base_ptr = bitcast ptr %base to ptr146 %res = call { <vscale x 2 x i64>, <vscale x 2 x i64> } @llvm.aarch64.sve.ld2.sret.nxv2i64(<vscale x 2 x i1> %Pg, ptr %base_ptr)147 ret { <vscale x 2 x i64>, <vscale x 2 x i64> } %res148}149 150define { <vscale x 2 x double>, <vscale x 2 x double> } @ld2.nxv4f64(<vscale x 2 x i1> %Pg, ptr %addr) {151; CHECK-LABEL: ld2.nxv4f64:152; CHECK: // %bb.0:153; CHECK-NEXT: ld2d { z0.d, z1.d }, p0/z, [x0, #-16, mul vl]154; CHECK-NEXT: ret155 %base = getelementptr <vscale x 2 x double>, ptr %addr, i64 -16156 %base_ptr = bitcast ptr %base to ptr157 %res = call { <vscale x 2 x double>, <vscale x 2 x double> } @llvm.aarch64.sve.ld2.sret.nxv2f64(<vscale x 2 x i1> %Pg, ptr %base_ptr)158 ret { <vscale x 2 x double>, <vscale x 2 x double> } %res159}160 161; ld3b162define { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } @ld3.nxv48i8(<vscale x 16 x i1> %Pg, ptr %addr) {163; CHECK-LABEL: ld3.nxv48i8:164; CHECK: // %bb.0:165; CHECK-NEXT: ld3b { z0.b - z2.b }, p0/z, [x0, #3, mul vl]166; CHECK-NEXT: ret167 %base = getelementptr <vscale x 16 x i8>, ptr %addr, i64 3168 %base_ptr = bitcast ptr %base to ptr169 %res = call { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } @llvm.aarch64.sve.ld3.sret.nxv16i8(<vscale x 16 x i1> %Pg, ptr %base_ptr)170 ret { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } %res171}172 173define { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } @ld3.nxv48i8_lower_bound(<vscale x 16 x i1> %Pg, ptr %addr) {174; CHECK-LABEL: ld3.nxv48i8_lower_bound:175; CHECK: // %bb.0:176; CHECK-NEXT: ld3b { z0.b - z2.b }, p0/z, [x0, #-24, mul vl]177; CHECK-NEXT: ret178 %base = getelementptr <vscale x 16 x i8>, ptr %addr, i64 -24179 %base_ptr = bitcast ptr %base to ptr180 %res = call { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } @llvm.aarch64.sve.ld3.sret.nxv16i8(<vscale x 16 x i1> %Pg, ptr %base_ptr)181 ret { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } %res182}183 184define { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } @ld3.nxv48i8_upper_bound(<vscale x 16 x i1> %Pg, ptr %addr) {185; CHECK-LABEL: ld3.nxv48i8_upper_bound:186; CHECK: // %bb.0:187; CHECK-NEXT: ld3b { z0.b - z2.b }, p0/z, [x0, #21, mul vl]188; CHECK-NEXT: ret189 %base = getelementptr <vscale x 16 x i8>, ptr %addr, i64 21190 %base_ptr = bitcast ptr %base to ptr191 %res = call { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } @llvm.aarch64.sve.ld3.sret.nxv16i8(<vscale x 16 x i1> %Pg, ptr %base_ptr)192 ret { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } %res193}194 195define { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } @ld3.nxv48i8_not_multiple_of_3_01(<vscale x 16 x i1> %Pg, ptr %addr) {196; CHECK-LABEL: ld3.nxv48i8_not_multiple_of_3_01:197; CHECK: // %bb.0:198; CHECK-NEXT: rdvl x8, #4199; CHECK-NEXT: ld3b { z0.b - z2.b }, p0/z, [x0, x8]200; CHECK-NEXT: ret201 %base = getelementptr <vscale x 16 x i8>, ptr %addr, i64 4202 %base_ptr = bitcast ptr %base to ptr203 %res = call { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } @llvm.aarch64.sve.ld3.sret.nxv16i8(<vscale x 16 x i1> %Pg, ptr %base_ptr)204 ret { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } %res205}206 207define { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } @ld3.nxv48i8_not_multiple_of_3_02(<vscale x 16 x i1> %Pg, ptr %addr) {208; CHECK-LABEL: ld3.nxv48i8_not_multiple_of_3_02:209; CHECK: // %bb.0:210; CHECK-NEXT: rdvl x8, #5211; CHECK-NEXT: ld3b { z0.b - z2.b }, p0/z, [x0, x8]212; CHECK-NEXT: ret213 %base = getelementptr <vscale x 16 x i8>, ptr %addr, i64 5214 %base_ptr = bitcast ptr %base to ptr215 %res = call { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } @llvm.aarch64.sve.ld3.sret.nxv16i8(<vscale x 16 x i1> %Pg, ptr %base_ptr)216 ret { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } %res217}218 219define { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } @ld3.nxv48i8_outside_lower_bound(<vscale x 16 x i1> %Pg, ptr %addr) {220; CHECK-LABEL: ld3.nxv48i8_outside_lower_bound:221; CHECK: // %bb.0:222; CHECK-NEXT: rdvl x8, #-27223; CHECK-NEXT: ld3b { z0.b - z2.b }, p0/z, [x0, x8]224; CHECK-NEXT: ret225 %base = getelementptr <vscale x 16 x i8>, ptr %addr, i64 -27226 %base_ptr = bitcast ptr %base to ptr227 %res = call { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } @llvm.aarch64.sve.ld3.sret.nxv16i8(<vscale x 16 x i1> %Pg, ptr %base_ptr)228 ret { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } %res229}230 231define { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } @ld3.nxv48i8_outside_upper_bound(<vscale x 16 x i1> %Pg, ptr %addr) {232; CHECK-LABEL: ld3.nxv48i8_outside_upper_bound:233; CHECK: // %bb.0:234; CHECK-NEXT: rdvl x8, #24235; CHECK-NEXT: ld3b { z0.b - z2.b }, p0/z, [x0, x8]236; CHECK-NEXT: ret237 %base = getelementptr <vscale x 16 x i8>, ptr %addr, i64 24238 %base_ptr = bitcast ptr %base to ptr239 %res = call { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } @llvm.aarch64.sve.ld3.sret.nxv16i8(<vscale x 16 x i1> %Pg, ptr %base_ptr)240 ret { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } %res241}242 243; ld3h244define { <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16> } @ld3.nxv24i16(<vscale x 8 x i1> %Pg, ptr %addr) {245; CHECK-LABEL: ld3.nxv24i16:246; CHECK: // %bb.0:247; CHECK-NEXT: ld3h { z0.h - z2.h }, p0/z, [x0, #21, mul vl]248; CHECK-NEXT: ret249 %base = getelementptr <vscale x 8 x i16>, ptr %addr, i64 21250 %base_ptr = bitcast ptr %base to ptr251 %res = call { <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16> } @llvm.aarch64.sve.ld3.sret.nxv8i16(<vscale x 8 x i1> %Pg, ptr %base_ptr)252 ret { <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16> } %res253}254 255define { <vscale x 8 x half>, <vscale x 8 x half>, <vscale x 8 x half> } @ld3.nxv24f16(<vscale x 8 x i1> %Pg, ptr %addr) {256; CHECK-LABEL: ld3.nxv24f16:257; CHECK: // %bb.0:258; CHECK-NEXT: ld3h { z0.h - z2.h }, p0/z, [x0, #21, mul vl]259; CHECK-NEXT: ret260 %base = getelementptr <vscale x 8 x half>, ptr %addr, i64 21261 %base_ptr = bitcast ptr %base to ptr262 %res = call { <vscale x 8 x half>, <vscale x 8 x half>, <vscale x 8 x half> } @llvm.aarch64.sve.ld3.sret.nxv8f16(<vscale x 8 x i1> %Pg, ptr %base_ptr)263 ret { <vscale x 8 x half>, <vscale x 8 x half>, <vscale x 8 x half> } %res264}265 266define { <vscale x 8 x bfloat>, <vscale x 8 x bfloat>, <vscale x 8 x bfloat> } @ld3.nxv24bf16(<vscale x 8 x i1> %Pg, ptr %addr) #0 {267; CHECK-LABEL: ld3.nxv24bf16:268; CHECK: // %bb.0:269; CHECK-NEXT: ld3h { z0.h - z2.h }, p0/z, [x0, #-24, mul vl]270; CHECK-NEXT: ret271 %base = getelementptr <vscale x 8 x bfloat>, ptr %addr, i64 -24272 %base_ptr = bitcast ptr %base to ptr273 %res = call { <vscale x 8 x bfloat>, <vscale x 8 x bfloat>, <vscale x 8 x bfloat> } @llvm.aarch64.sve.ld3.sret.nxv8bf16(<vscale x 8 x i1> %Pg, ptr %base_ptr)274 ret { <vscale x 8 x bfloat>, <vscale x 8 x bfloat>, <vscale x 8 x bfloat> } %res275}276 277; ld3w278define { <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> } @ld3.nxv12i32(<vscale x 4 x i1> %Pg, ptr %addr) {279; CHECK-LABEL: ld3.nxv12i32:280; CHECK: // %bb.0:281; CHECK-NEXT: ld3w { z0.s - z2.s }, p0/z, [x0, #21, mul vl]282; CHECK-NEXT: ret283 %base = getelementptr <vscale x 4 x i32>, ptr %addr, i64 21284 %base_ptr = bitcast ptr %base to ptr285 %res = call { <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> } @llvm.aarch64.sve.ld3.sret.nxv4i32(<vscale x 4 x i1> %Pg, ptr %base_ptr)286 ret { <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> } %res287}288 289define { <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float> } @ld3.nxv12f32(<vscale x 4 x i1> %Pg, ptr %addr) {290; CHECK-LABEL: ld3.nxv12f32:291; CHECK: // %bb.0:292; CHECK-NEXT: ld3w { z0.s - z2.s }, p0/z, [x0, #-24, mul vl]293; CHECK-NEXT: ret294 %base = getelementptr <vscale x 4 x float>, ptr %addr, i64 -24295 %base_ptr = bitcast ptr %base to ptr296 %res = call { <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float> } @llvm.aarch64.sve.ld3.sret.nxv4f32(<vscale x 4 x i1> %Pg, ptr %base_ptr)297 ret { <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float> } %res298}299 300; ld3d301define { <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64> } @ld3.nxv6i64(<vscale x 2 x i1> %Pg, ptr %addr) {302; CHECK-LABEL: ld3.nxv6i64:303; CHECK: // %bb.0:304; CHECK-NEXT: ld3d { z0.d - z2.d }, p0/z, [x0, #21, mul vl]305; CHECK-NEXT: ret306 %base = getelementptr <vscale x 2 x i64>, ptr %addr, i64 21307 %base_ptr = bitcast ptr %base to ptr308 %res = call { <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64> } @llvm.aarch64.sve.ld3.sret.nxv2i64(<vscale x 2 x i1> %Pg, ptr %base_ptr)309 ret { <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64> } %res310}311 312define { <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double> } @ld3.nxv6f64(<vscale x 2 x i1> %Pg, ptr %addr) {313; CHECK-LABEL: ld3.nxv6f64:314; CHECK: // %bb.0:315; CHECK-NEXT: ld3d { z0.d - z2.d }, p0/z, [x0, #-24, mul vl]316; CHECK-NEXT: ret317 %base = getelementptr <vscale x 2 x double>, ptr %addr, i64 -24318 %base_ptr = bitcast ptr %base to ptr319 %res = call { <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double> } @llvm.aarch64.sve.ld3.sret.nxv2f64(<vscale x 2 x i1> %Pg, ptr %base_ptr)320 ret { <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double> } %res321}322 323; ; ld4b324define { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } @ld4.nxv64i8(<vscale x 16 x i1> %Pg, ptr %addr) {325; CHECK-LABEL: ld4.nxv64i8:326; CHECK: // %bb.0:327; CHECK-NEXT: ld4b { z0.b - z3.b }, p0/z, [x0, #4, mul vl]328; CHECK-NEXT: ret329 %base = getelementptr <vscale x 16 x i8>, ptr %addr, i64 4330 %base_ptr = bitcast ptr %base to ptr331 %res = call { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } @llvm.aarch64.sve.ld4.sret.nxv16i8(<vscale x 16 x i1> %Pg, ptr %base_ptr)332 ret { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } %res333}334 335define { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } @ld4.nxv64i8_lower_bound(<vscale x 16 x i1> %Pg, ptr %addr) {336; CHECK-LABEL: ld4.nxv64i8_lower_bound:337; CHECK: // %bb.0:338; CHECK-NEXT: ld4b { z0.b - z3.b }, p0/z, [x0, #-32, mul vl]339; CHECK-NEXT: ret340 %base = getelementptr <vscale x 16 x i8>, ptr %addr, i64 -32341 %base_ptr = bitcast ptr %base to ptr342 %res = call { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } @llvm.aarch64.sve.ld4.sret.nxv16i8(<vscale x 16 x i1> %Pg, ptr %base_ptr)343 ret { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } %res344}345 346define { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } @ld4.nxv64i8_upper_bound(<vscale x 16 x i1> %Pg, ptr %addr) {347; CHECK-LABEL: ld4.nxv64i8_upper_bound:348; CHECK: // %bb.0:349; CHECK-NEXT: ld4b { z0.b - z3.b }, p0/z, [x0, #28, mul vl]350; CHECK-NEXT: ret351 %base = getelementptr <vscale x 16 x i8>, ptr %addr, i64 28352 %base_ptr = bitcast ptr %base to ptr353 %res = call { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } @llvm.aarch64.sve.ld4.sret.nxv16i8(<vscale x 16 x i1> %Pg, ptr %base_ptr)354 ret { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } %res355}356 357define { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } @ld4.nxv64i8_not_multiple_of_4_01(<vscale x 16 x i1> %Pg, ptr %addr) {358; CHECK-LABEL: ld4.nxv64i8_not_multiple_of_4_01:359; CHECK: // %bb.0:360; CHECK-NEXT: rdvl x8, #5361; CHECK-NEXT: ld4b { z0.b - z3.b }, p0/z, [x0, x8]362; CHECK-NEXT: ret363 %base = getelementptr <vscale x 16 x i8>, ptr %addr, i64 5364 %base_ptr = bitcast ptr %base to ptr365 %res = call { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } @llvm.aarch64.sve.ld4.sret.nxv16i8(<vscale x 16 x i1> %Pg, ptr %base_ptr)366 ret { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } %res367}368 369define { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } @ld4.nxv64i8_not_multiple_of_4_02(<vscale x 16 x i1> %Pg, ptr %addr) {370; CHECK-LABEL: ld4.nxv64i8_not_multiple_of_4_02:371; CHECK: // %bb.0:372; CHECK-NEXT: rdvl x8, #6373; CHECK-NEXT: ld4b { z0.b - z3.b }, p0/z, [x0, x8]374; CHECK-NEXT: ret375 %base = getelementptr <vscale x 16 x i8>, ptr %addr, i64 6376 %base_ptr = bitcast ptr %base to ptr377 %res = call { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } @llvm.aarch64.sve.ld4.sret.nxv16i8(<vscale x 16 x i1> %Pg, ptr %base_ptr)378 ret { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } %res379}380 381define { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } @ld4.nxv64i8_not_multiple_of_4_03(<vscale x 16 x i1> %Pg, ptr %addr) {382; CHECK-LABEL: ld4.nxv64i8_not_multiple_of_4_03:383; CHECK: // %bb.0:384; CHECK-NEXT: rdvl x8, #7385; CHECK-NEXT: ld4b { z0.b - z3.b }, p0/z, [x0, x8]386; CHECK-NEXT: ret387 %base = getelementptr <vscale x 16 x i8>, ptr %addr, i64 7388 %base_ptr = bitcast ptr %base to ptr389 %res = call { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } @llvm.aarch64.sve.ld4.sret.nxv16i8(<vscale x 16 x i1> %Pg, ptr %base_ptr)390 ret { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } %res391}392 393define { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } @ld4.nxv64i8_outside_lower_bound(<vscale x 16 x i1> %Pg, ptr %addr) {394; CHECK-LABEL: ld4.nxv64i8_outside_lower_bound:395; CHECK: // %bb.0:396; CHECK-NEXT: rdvl x8, #1397; CHECK-NEXT: mov x9, #-576398; CHECK-NEXT: lsr x8, x8, #4399; CHECK-NEXT: mul x8, x8, x9400; CHECK-NEXT: ld4b { z0.b - z3.b }, p0/z, [x0, x8]401; CHECK-NEXT: ret402; FIXME: optimize OFFSET computation so that xOFFSET = (mul (RDVL #4) #9)403; xM = -9 * 2^6404; xP = RDVL * 2^-4405; xOFFSET = RDVL * 2^-4 * -9 * 2^6 = RDVL * -36406 %base = getelementptr <vscale x 16 x i8>, ptr %addr, i64 -36407 %base_ptr = bitcast ptr %base to ptr408 %res = call { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } @llvm.aarch64.sve.ld4.sret.nxv16i8(<vscale x 16 x i1> %Pg, ptr %base_ptr)409 ret { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } %res410}411 412define { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } @ld4.nxv64i8_outside_upper_bound(<vscale x 16 x i1> %Pg, ptr %addr) {413; CHECK-LABEL: ld4.nxv64i8_outside_upper_bound:414; CHECK: // %bb.0:415; CHECK-NEXT: rdvl x8, #1416; CHECK-NEXT: mov w9, #512417; CHECK-NEXT: lsr x8, x8, #4418; CHECK-NEXT: mul x8, x8, x9419; CHECK-NEXT: ld4b { z0.b - z3.b }, p0/z, [x0, x8]420; CHECK-NEXT: ret421; FIXME: optimize OFFSET computation so that xOFFSET = (mul (RDVL #16) #2)422; xM = 2^9423; xP = RDVL * 2^-4424; xOFFSET = RDVL * 2^-4 * 2^9 = RDVL * 32425 %base = getelementptr <vscale x 16 x i8>, ptr %addr, i64 32426 %base_ptr = bitcast ptr %base to ptr427 %res = call { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } @llvm.aarch64.sve.ld4.sret.nxv16i8(<vscale x 16 x i1> %Pg, ptr %base_ptr)428 ret { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } %res429}430 431; ld4h432define { <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16> } @ld4.nxv32i16(<vscale x 8 x i1> %Pg, ptr %addr) {433; CHECK-LABEL: ld4.nxv32i16:434; CHECK: // %bb.0:435; CHECK-NEXT: ld4h { z0.h - z3.h }, p0/z, [x0, #8, mul vl]436; CHECK-NEXT: ret437 %base = getelementptr <vscale x 8 x i16>, ptr %addr, i64 8438 %base_ptr = bitcast ptr %base to ptr439 %res = call { <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16> } @llvm.aarch64.sve.ld4.sret.nxv8i16(<vscale x 8 x i1> %Pg, ptr %base_ptr)440 ret { <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16> } %res441}442 443define { <vscale x 8 x half>, <vscale x 8 x half>, <vscale x 8 x half>, <vscale x 8 x half> } @ld4.nxv32f16(<vscale x 8 x i1> %Pg, ptr %addr) {444; CHECK-LABEL: ld4.nxv32f16:445; CHECK: // %bb.0:446; CHECK-NEXT: ld4h { z0.h - z3.h }, p0/z, [x0, #28, mul vl]447; CHECK-NEXT: ret448 %base = getelementptr <vscale x 8 x half>, ptr %addr, i64 28449 %base_ptr = bitcast ptr %base to ptr450 %res = call { <vscale x 8 x half>, <vscale x 8 x half>, <vscale x 8 x half>, <vscale x 8 x half> } @llvm.aarch64.sve.ld4.sret.nxv8f16(<vscale x 8 x i1> %Pg, ptr %base_ptr)451 ret { <vscale x 8 x half>, <vscale x 8 x half>, <vscale x 8 x half>, <vscale x 8 x half> } %res452}453 454define { <vscale x 8 x bfloat>, <vscale x 8 x bfloat>, <vscale x 8 x bfloat>, <vscale x 8 x bfloat> } @ld4.nxv32bf16(<vscale x 8 x i1> %Pg, ptr %addr) #0 {455; CHECK-LABEL: ld4.nxv32bf16:456; CHECK: // %bb.0:457; CHECK-NEXT: ld4h { z0.h - z3.h }, p0/z, [x0, #-32, mul vl]458; CHECK-NEXT: ret459 %base = getelementptr <vscale x 8 x bfloat>, ptr %addr, i64 -32460 %base_ptr = bitcast ptr %base to ptr461 %res = call { <vscale x 8 x bfloat>, <vscale x 8 x bfloat>, <vscale x 8 x bfloat>, <vscale x 8 x bfloat> } @llvm.aarch64.sve.ld4.sret.nxv8bf16(<vscale x 8 x i1> %Pg, ptr %base_ptr)462 ret { <vscale x 8 x bfloat>, <vscale x 8 x bfloat>, <vscale x 8 x bfloat>, <vscale x 8 x bfloat> } %res463}464 465; ld4w466define { <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> } @ld4.nxv16i32(<vscale x 4 x i1> %Pg, ptr %addr) {467; CHECK-LABEL: ld4.nxv16i32:468; CHECK: // %bb.0:469; CHECK-NEXT: ld4w { z0.s - z3.s }, p0/z, [x0, #28, mul vl]470; CHECK-NEXT: ret471 %base = getelementptr <vscale x 4 x i32>, ptr %addr, i64 28472 %base_ptr = bitcast ptr %base to ptr473 %res = call { <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> } @llvm.aarch64.sve.ld4.sret.nxv4i32(<vscale x 4 x i1> %Pg, ptr %base_ptr)474 ret { <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> } %res475}476 477define { <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float> } @ld4.nxv16f32(<vscale x 4 x i1> %Pg, ptr %addr) {478; CHECK-LABEL: ld4.nxv16f32:479; CHECK: // %bb.0:480; CHECK-NEXT: ld4w { z0.s - z3.s }, p0/z, [x0, #-32, mul vl]481; CHECK-NEXT: ret482 %base = getelementptr <vscale x 4 x float>, ptr %addr, i64 -32483 %base_ptr = bitcast ptr %base to ptr484 %res = call { <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float> } @llvm.aarch64.sve.ld4.sret.nxv4f32(<vscale x 4 x i1> %Pg, ptr %base_ptr)485 ret { <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float> } %res486}487 488; ld4d489define { <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64> } @ld4.nxv8i64(<vscale x 2 x i1> %Pg, ptr %addr) {490; CHECK-LABEL: ld4.nxv8i64:491; CHECK: // %bb.0:492; CHECK-NEXT: ld4d { z0.d - z3.d }, p0/z, [x0, #28, mul vl]493; CHECK-NEXT: ret494 %base = getelementptr <vscale x 2 x i64>, ptr %addr, i64 28495 %base_ptr = bitcast ptr %base to ptr496 %res = call { <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64> } @llvm.aarch64.sve.ld4.sret.nxv2i64(<vscale x 2 x i1> %Pg, ptr %base_ptr)497 ret { <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64> } %res498}499 500define { <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double> } @ld4.nxv8f64(<vscale x 2 x i1> %Pg, ptr %addr) {501; CHECK-LABEL: ld4.nxv8f64:502; CHECK: // %bb.0:503; CHECK-NEXT: ld4d { z0.d - z3.d }, p0/z, [x0, #-32, mul vl]504; CHECK-NEXT: ret505 %base = getelementptr <vscale x 2 x double>, ptr %addr, i64 -32506 %base_ptr = bitcast ptr %base to ptr507 %res = call { <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double> } @llvm.aarch64.sve.ld4.sret.nxv2f64(<vscale x 2 x i1> %Pg, ptr %base_ptr)508 ret { <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double> } %res509}510 511declare { <vscale x 16 x i8>, <vscale x 16 x i8> } @llvm.aarch64.sve.ld2.sret.nxv16i8(<vscale x 16 x i1>, ptr)512declare { <vscale x 8 x i16>, <vscale x 8 x i16> } @llvm.aarch64.sve.ld2.sret.nxv8i16(<vscale x 8 x i1>, ptr)513declare { <vscale x 4 x i32>, <vscale x 4 x i32> } @llvm.aarch64.sve.ld2.sret.nxv4i32(<vscale x 4 x i1>, ptr)514declare { <vscale x 2 x i64>, <vscale x 2 x i64> } @llvm.aarch64.sve.ld2.sret.nxv2i64(<vscale x 2 x i1>, ptr)515declare { <vscale x 8 x half>, <vscale x 8 x half> } @llvm.aarch64.sve.ld2.sret.nxv8f16(<vscale x 8 x i1>, ptr)516declare { <vscale x 8 x bfloat>, <vscale x 8 x bfloat> } @llvm.aarch64.sve.ld2.sret.nxv8bf16(<vscale x 8 x i1>, ptr)517declare { <vscale x 4 x float>, <vscale x 4 x float> } @llvm.aarch64.sve.ld2.sret.nxv4f32(<vscale x 4 x i1>, ptr)518declare { <vscale x 2 x double>, <vscale x 2 x double> } @llvm.aarch64.sve.ld2.sret.nxv2f64(<vscale x 2 x i1>, ptr)519 520declare { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } @llvm.aarch64.sve.ld3.sret.nxv16i8(<vscale x 16 x i1>, ptr)521declare { <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16> } @llvm.aarch64.sve.ld3.sret.nxv8i16(<vscale x 8 x i1>, ptr)522declare { <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> } @llvm.aarch64.sve.ld3.sret.nxv4i32(<vscale x 4 x i1>, ptr)523declare { <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64> } @llvm.aarch64.sve.ld3.sret.nxv2i64(<vscale x 2 x i1>, ptr)524declare { <vscale x 8 x half>, <vscale x 8 x half>, <vscale x 8 x half> } @llvm.aarch64.sve.ld3.sret.nxv8f16(<vscale x 8 x i1>, ptr)525declare { <vscale x 8 x bfloat>, <vscale x 8 x bfloat>, <vscale x 8 x bfloat> } @llvm.aarch64.sve.ld3.sret.nxv8bf16(<vscale x 8 x i1>, ptr)526declare { <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float> } @llvm.aarch64.sve.ld3.sret.nxv4f32(<vscale x 4 x i1>, ptr)527declare { <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double> } @llvm.aarch64.sve.ld3.sret.nxv2f64(<vscale x 2 x i1>, ptr)528 529declare { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } @llvm.aarch64.sve.ld4.sret.nxv16i8(<vscale x 16 x i1>, ptr)530declare { <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16> } @llvm.aarch64.sve.ld4.sret.nxv8i16(<vscale x 8 x i1>, ptr)531declare { <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> } @llvm.aarch64.sve.ld4.sret.nxv4i32(<vscale x 4 x i1>, ptr)532declare { <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64> } @llvm.aarch64.sve.ld4.sret.nxv2i64(<vscale x 2 x i1>, ptr)533declare { <vscale x 8 x half>, <vscale x 8 x half>, <vscale x 8 x half>, <vscale x 8 x half> } @llvm.aarch64.sve.ld4.sret.nxv8f16(<vscale x 8 x i1>, ptr)534declare { <vscale x 8 x bfloat>, <vscale x 8 x bfloat>, <vscale x 8 x bfloat>, <vscale x 8 x bfloat> } @llvm.aarch64.sve.ld4.sret.nxv8bf16(<vscale x 8 x i1>, ptr)535declare { <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float> } @llvm.aarch64.sve.ld4.sret.nxv4f32(<vscale x 4 x i1>, ptr)536declare { <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double>, <vscale x 2 x double> } @llvm.aarch64.sve.ld4.sret.nxv2f64(<vscale x 2 x i1>, ptr)537 538; +bf16 is required for the bfloat version.539attributes #0 = { "target-features"="+bf16" }540