660 lines · plain
1; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py2; RUN: llc -aarch64-sve-vector-bits-min=256 -aarch64-sve-vector-bits-max=256 < %s | FileCheck %s -check-prefixes=CHECK,VBITS_EQ_2563; RUN: llc -aarch64-sve-vector-bits-min=512 -aarch64-sve-vector-bits-max=512 < %s | FileCheck %s -check-prefixes=CHECK,VBITS_EQ_5124 5target triple = "aarch64-unknown-linux-gnu"6 7define void @zip1_v32i8(ptr %a, ptr %b) #0 {8; VBITS_EQ_256-LABEL: zip1_v32i8:9; VBITS_EQ_256: // %bb.0:10; VBITS_EQ_256-NEXT: ldr z0, [x0]11; VBITS_EQ_256-NEXT: ldr z1, [x1]12; VBITS_EQ_256-NEXT: zip1 z0.b, z0.b, z1.b13; VBITS_EQ_256-NEXT: str z0, [x0]14; VBITS_EQ_256-NEXT: ret15;16; VBITS_EQ_512-LABEL: zip1_v32i8:17; VBITS_EQ_512: // %bb.0:18; VBITS_EQ_512-NEXT: ptrue p0.b, vl3219; VBITS_EQ_512-NEXT: ld1b { z0.b }, p0/z, [x0]20; VBITS_EQ_512-NEXT: ld1b { z1.b }, p0/z, [x1]21; VBITS_EQ_512-NEXT: zip1 z0.b, z0.b, z1.b22; VBITS_EQ_512-NEXT: st1b { z0.b }, p0, [x0]23; VBITS_EQ_512-NEXT: ret24 %tmp1 = load volatile <32 x i8>, ptr %a25 %tmp2 = load volatile <32 x i8>, ptr %b26 %tmp3 = shufflevector <32 x i8> %tmp1, <32 x i8> %tmp2, <32 x i32> <i32 0, i32 32, i32 1, i32 33, i32 2, i32 34, i32 3, i32 35, i32 4, i32 36, i32 5, i32 37, i32 6, i32 38, i32 7, i32 39, i32 8, i32 40, i32 9, i32 41, i32 10, i32 42, i32 11, i32 43, i32 12, i32 44, i32 13, i32 45, i32 14, i32 46, i32 15, i32 47>27 store volatile <32 x i8> %tmp3, ptr %a28 ret void29}30 31define void @zip_v32i16(ptr %a, ptr %b) #0 {32; VBITS_EQ_256-LABEL: zip_v32i16:33; VBITS_EQ_256: // %bb.0:34; VBITS_EQ_256-NEXT: ldr z0, [x0, #1, mul vl]35; VBITS_EQ_256-NEXT: ldr z1, [x0]36; VBITS_EQ_256-NEXT: ldr z2, [x1, #1, mul vl]37; VBITS_EQ_256-NEXT: ldr z3, [x1]38; VBITS_EQ_256-NEXT: zip1 z5.h, z0.h, z2.h39; VBITS_EQ_256-NEXT: zip2 z0.h, z0.h, z2.h40; VBITS_EQ_256-NEXT: zip1 z4.h, z1.h, z3.h41; VBITS_EQ_256-NEXT: zip2 z1.h, z1.h, z3.h42; VBITS_EQ_256-NEXT: add z2.h, z4.h, z5.h43; VBITS_EQ_256-NEXT: add z0.h, z1.h, z0.h44; VBITS_EQ_256-NEXT: str z2, [x0]45; VBITS_EQ_256-NEXT: str z0, [x0, #1, mul vl]46; VBITS_EQ_256-NEXT: ret47;48; VBITS_EQ_512-LABEL: zip_v32i16:49; VBITS_EQ_512: // %bb.0:50; VBITS_EQ_512-NEXT: ldr z0, [x0]51; VBITS_EQ_512-NEXT: ldr z1, [x1]52; VBITS_EQ_512-NEXT: zip1 z2.h, z0.h, z1.h53; VBITS_EQ_512-NEXT: zip2 z0.h, z0.h, z1.h54; VBITS_EQ_512-NEXT: add z0.h, z2.h, z0.h55; VBITS_EQ_512-NEXT: str z0, [x0]56; VBITS_EQ_512-NEXT: ret57 %tmp1 = load <32 x i16>, ptr %a58 %tmp2 = load <32 x i16>, ptr %b59 %tmp3 = shufflevector <32 x i16> %tmp1, <32 x i16> %tmp2, <32 x i32> <i32 0, i32 32, i32 1, i32 33, i32 2, i32 34, i32 3, i32 35, i32 4, i32 36, i32 5, i32 37, i32 6, i32 38, i32 7, i32 39, i32 8, i32 40, i32 9, i32 41, i32 10, i32 42, i32 11, i32 43, i32 12, i32 44, i32 13, i32 45, i32 14, i32 46, i32 15, i32 47>60 %tmp4 = shufflevector <32 x i16> %tmp1, <32 x i16> %tmp2, <32 x i32> <i32 16, i32 48, i32 17, i32 49, i32 18, i32 50, i32 19, i32 51, i32 20, i32 52, i32 21, i32 53, i32 22, i32 54, i32 23, i32 55, i32 24, i32 56, i32 25, i32 57, i32 26, i32 58, i32 27, i32 59, i32 28, i32 60, i32 29, i32 61, i32 30, i32 62, i32 31, i32 63>61 %tmp5 = add <32 x i16> %tmp3, %tmp462 store <32 x i16> %tmp5, ptr %a63 ret void64}65 66define void @zip1_v16i16(ptr %a, ptr %b) #0 {67; VBITS_EQ_256-LABEL: zip1_v16i16:68; VBITS_EQ_256: // %bb.0:69; VBITS_EQ_256-NEXT: ldr z0, [x0]70; VBITS_EQ_256-NEXT: ldr z1, [x1]71; VBITS_EQ_256-NEXT: zip1 z0.h, z0.h, z1.h72; VBITS_EQ_256-NEXT: str z0, [x0]73; VBITS_EQ_256-NEXT: ret74;75; VBITS_EQ_512-LABEL: zip1_v16i16:76; VBITS_EQ_512: // %bb.0:77; VBITS_EQ_512-NEXT: ptrue p0.h, vl1678; VBITS_EQ_512-NEXT: ld1h { z0.h }, p0/z, [x0]79; VBITS_EQ_512-NEXT: ld1h { z1.h }, p0/z, [x1]80; VBITS_EQ_512-NEXT: zip1 z0.h, z0.h, z1.h81; VBITS_EQ_512-NEXT: st1h { z0.h }, p0, [x0]82; VBITS_EQ_512-NEXT: ret83 %tmp1 = load volatile <16 x i16>, ptr %a84 %tmp2 = load volatile <16 x i16>, ptr %b85 %tmp3 = shufflevector <16 x i16> %tmp1, <16 x i16> %tmp2, <16 x i32> <i32 0, i32 16, i32 1, i32 17, i32 2, i32 18, i32 3, i32 19, i32 4, i32 20, i32 5, i32 21, i32 6, i32 22, i32 7, i32 23>86 store volatile <16 x i16> %tmp3, ptr %a87 ret void88}89 90define void @zip1_v8i32(ptr %a, ptr %b) #0 {91; VBITS_EQ_256-LABEL: zip1_v8i32:92; VBITS_EQ_256: // %bb.0:93; VBITS_EQ_256-NEXT: ldr z0, [x0]94; VBITS_EQ_256-NEXT: ldr z1, [x1]95; VBITS_EQ_256-NEXT: zip1 z0.s, z0.s, z1.s96; VBITS_EQ_256-NEXT: str z0, [x0]97; VBITS_EQ_256-NEXT: ret98;99; VBITS_EQ_512-LABEL: zip1_v8i32:100; VBITS_EQ_512: // %bb.0:101; VBITS_EQ_512-NEXT: ptrue p0.s, vl8102; VBITS_EQ_512-NEXT: ld1w { z0.s }, p0/z, [x0]103; VBITS_EQ_512-NEXT: ld1w { z1.s }, p0/z, [x1]104; VBITS_EQ_512-NEXT: zip1 z0.s, z0.s, z1.s105; VBITS_EQ_512-NEXT: st1w { z0.s }, p0, [x0]106; VBITS_EQ_512-NEXT: ret107 %tmp1 = load volatile <8 x i32>, ptr %a108 %tmp2 = load volatile <8 x i32>, ptr %b109 %tmp3 = shufflevector <8 x i32> %tmp1, <8 x i32> %tmp2, <8 x i32> <i32 0, i32 8, i32 1, i32 9, i32 2, i32 10, i32 3, i32 11>110 store volatile <8 x i32> %tmp3, ptr %a111 ret void112}113 114define void @zip_v4f64(ptr %a, ptr %b) #0 {115; VBITS_EQ_256-LABEL: zip_v4f64:116; VBITS_EQ_256: // %bb.0:117; VBITS_EQ_256-NEXT: ldr z0, [x0]118; VBITS_EQ_256-NEXT: ldr z1, [x1]119; VBITS_EQ_256-NEXT: zip1 z2.d, z0.d, z1.d120; VBITS_EQ_256-NEXT: zip2 z0.d, z0.d, z1.d121; VBITS_EQ_256-NEXT: fadd z0.d, z2.d, z0.d122; VBITS_EQ_256-NEXT: str z0, [x0]123; VBITS_EQ_256-NEXT: ret124;125; VBITS_EQ_512-LABEL: zip_v4f64:126; VBITS_EQ_512: // %bb.0:127; VBITS_EQ_512-NEXT: stp x29, x30, [sp, #-16]! // 16-byte Folded Spill128; VBITS_EQ_512-NEXT: sub x9, sp, #48129; VBITS_EQ_512-NEXT: mov x29, sp130; VBITS_EQ_512-NEXT: and sp, x9, #0xffffffffffffffe0131; VBITS_EQ_512-NEXT: .cfi_def_cfa w29, 16132; VBITS_EQ_512-NEXT: .cfi_offset w30, -8133; VBITS_EQ_512-NEXT: .cfi_offset w29, -16134; VBITS_EQ_512-NEXT: ptrue p0.d, vl4135; VBITS_EQ_512-NEXT: mov x8, sp136; VBITS_EQ_512-NEXT: ld1d { z0.d }, p0/z, [x1]137; VBITS_EQ_512-NEXT: ld1d { z1.d }, p0/z, [x0]138; VBITS_EQ_512-NEXT: mov z2.d, z0.d[3]139; VBITS_EQ_512-NEXT: mov z3.d, z1.d[3]140; VBITS_EQ_512-NEXT: mov z4.d, z0.d[2]141; VBITS_EQ_512-NEXT: mov z5.d, z1.d[2]142; VBITS_EQ_512-NEXT: zip1 z0.d, z1.d, z0.d143; VBITS_EQ_512-NEXT: stp d3, d2, [sp, #16]144; VBITS_EQ_512-NEXT: stp d5, d4, [sp]145; VBITS_EQ_512-NEXT: ld1d { z1.d }, p0/z, [x8]146; VBITS_EQ_512-NEXT: fadd z0.d, p0/m, z0.d, z1.d147; VBITS_EQ_512-NEXT: st1d { z0.d }, p0, [x0]148; VBITS_EQ_512-NEXT: mov sp, x29149; VBITS_EQ_512-NEXT: ldp x29, x30, [sp], #16 // 16-byte Folded Reload150; VBITS_EQ_512-NEXT: ret151 %tmp1 = load <4 x double>, ptr %a152 %tmp2 = load <4 x double>, ptr %b153 %tmp3 = shufflevector <4 x double> %tmp1, <4 x double> %tmp2, <4 x i32> <i32 0, i32 4, i32 1, i32 5>154 %tmp4 = shufflevector <4 x double> %tmp1, <4 x double> %tmp2, <4 x i32> <i32 2, i32 6, i32 3, i32 7>155 %tmp5 = fadd <4 x double> %tmp3, %tmp4156 store <4 x double> %tmp5, ptr %a157 ret void158}159 160; Don't use SVE for 128-bit vectors161define void @zip_v4i32(ptr %a, ptr %b) #0 {162; CHECK-LABEL: zip_v4i32:163; CHECK: // %bb.0:164; CHECK-NEXT: ldr q0, [x0]165; CHECK-NEXT: ldr q1, [x1]166; CHECK-NEXT: zip1 v2.4s, v0.4s, v1.4s167; CHECK-NEXT: zip2 v0.4s, v0.4s, v1.4s168; CHECK-NEXT: add v0.4s, v2.4s, v0.4s169; CHECK-NEXT: str q0, [x0]170; CHECK-NEXT: ret171 %tmp1 = load <4 x i32>, ptr %a172 %tmp2 = load <4 x i32>, ptr %b173 %tmp3 = shufflevector <4 x i32> %tmp1, <4 x i32> %tmp2, <4 x i32> <i32 0, i32 4, i32 1, i32 5>174 %tmp4 = shufflevector <4 x i32> %tmp1, <4 x i32> %tmp2, <4 x i32> <i32 2, i32 6, i32 3, i32 7>175 %tmp5 = add <4 x i32> %tmp3, %tmp4176 store <4 x i32> %tmp5, ptr %a177 ret void178}179 180define void @zip1_v8i32_undef(ptr %a) #0 {181; VBITS_EQ_256-LABEL: zip1_v8i32_undef:182; VBITS_EQ_256: // %bb.0:183; VBITS_EQ_256-NEXT: ldr z0, [x0]184; VBITS_EQ_256-NEXT: zip1 z0.s, z0.s, z0.s185; VBITS_EQ_256-NEXT: str z0, [x0]186; VBITS_EQ_256-NEXT: ret187;188; VBITS_EQ_512-LABEL: zip1_v8i32_undef:189; VBITS_EQ_512: // %bb.0:190; VBITS_EQ_512-NEXT: ptrue p0.s, vl8191; VBITS_EQ_512-NEXT: ld1w { z0.s }, p0/z, [x0]192; VBITS_EQ_512-NEXT: zip1 z0.s, z0.s, z0.s193; VBITS_EQ_512-NEXT: st1w { z0.s }, p0, [x0]194; VBITS_EQ_512-NEXT: ret195 %tmp1 = load volatile <8 x i32>, ptr %a196 %tmp2 = shufflevector <8 x i32> %tmp1, <8 x i32> poison, <8 x i32> <i32 0, i32 0, i32 1, i32 1, i32 2, i32 2, i32 3, i32 3>197 store volatile <8 x i32> %tmp2, ptr %a198 ret void199}200 201define void @trn_v32i8(ptr %a, ptr %b) #0 {202; VBITS_EQ_256-LABEL: trn_v32i8:203; VBITS_EQ_256: // %bb.0:204; VBITS_EQ_256-NEXT: ldr z0, [x0]205; VBITS_EQ_256-NEXT: ldr z1, [x1]206; VBITS_EQ_256-NEXT: trn1 z2.b, z0.b, z1.b207; VBITS_EQ_256-NEXT: trn2 z0.b, z0.b, z1.b208; VBITS_EQ_256-NEXT: add z0.b, z2.b, z0.b209; VBITS_EQ_256-NEXT: str z0, [x0]210; VBITS_EQ_256-NEXT: ret211;212; VBITS_EQ_512-LABEL: trn_v32i8:213; VBITS_EQ_512: // %bb.0:214; VBITS_EQ_512-NEXT: ptrue p0.b, vl32215; VBITS_EQ_512-NEXT: ld1b { z0.b }, p0/z, [x0]216; VBITS_EQ_512-NEXT: ld1b { z1.b }, p0/z, [x1]217; VBITS_EQ_512-NEXT: trn1 z2.b, z0.b, z1.b218; VBITS_EQ_512-NEXT: trn2 z0.b, z0.b, z1.b219; VBITS_EQ_512-NEXT: add z0.b, z2.b, z0.b220; VBITS_EQ_512-NEXT: st1b { z0.b }, p0, [x0]221; VBITS_EQ_512-NEXT: ret222 %tmp1 = load <32 x i8>, ptr %a223 %tmp2 = load <32 x i8>, ptr %b224 %tmp3 = shufflevector <32 x i8> %tmp1, <32 x i8> %tmp2, <32 x i32> <i32 0, i32 32, i32 2, i32 34, i32 4, i32 36, i32 6, i32 38, i32 8, i32 40, i32 10, i32 42, i32 12, i32 44, i32 14, i32 46, i32 16, i32 48, i32 18, i32 50, i32 20, i32 52, i32 22, i32 54, i32 24, i32 56, i32 26, i32 58, i32 28, i32 60, i32 30, i32 62>225 %tmp4 = shufflevector <32 x i8> %tmp1, <32 x i8> %tmp2, <32 x i32> <i32 1, i32 33, i32 3, i32 35, i32 poison, i32 37, i32 7, i32 poison, i32 poison, i32 41, i32 11, i32 43, i32 13, i32 45, i32 15, i32 47, i32 17, i32 49, i32 19, i32 51, i32 21, i32 53, i32 23, i32 55, i32 25, i32 57, i32 27, i32 59, i32 29, i32 61, i32 31, i32 63>226 %tmp5 = add <32 x i8> %tmp3, %tmp4227 store <32 x i8> %tmp5, ptr %a228 ret void229}230 231define void @trn_v32i16(ptr %a, ptr %b) #0 {232; VBITS_EQ_256-LABEL: trn_v32i16:233; VBITS_EQ_256: // %bb.0:234; VBITS_EQ_256-NEXT: ldr z0, [x0]235; VBITS_EQ_256-NEXT: ldr z1, [x1]236; VBITS_EQ_256-NEXT: ldr z2, [x0, #1, mul vl]237; VBITS_EQ_256-NEXT: ldr z3, [x1, #1, mul vl]238; VBITS_EQ_256-NEXT: trn1 z4.h, z0.h, z1.h239; VBITS_EQ_256-NEXT: trn2 z0.h, z0.h, z1.h240; VBITS_EQ_256-NEXT: trn1 z1.h, z2.h, z3.h241; VBITS_EQ_256-NEXT: trn2 z2.h, z2.h, z3.h242; VBITS_EQ_256-NEXT: add z0.h, z4.h, z0.h243; VBITS_EQ_256-NEXT: add z1.h, z1.h, z2.h244; VBITS_EQ_256-NEXT: str z0, [x0]245; VBITS_EQ_256-NEXT: str z1, [x0, #1, mul vl]246; VBITS_EQ_256-NEXT: ret247;248; VBITS_EQ_512-LABEL: trn_v32i16:249; VBITS_EQ_512: // %bb.0:250; VBITS_EQ_512-NEXT: ldr z0, [x0]251; VBITS_EQ_512-NEXT: ldr z1, [x1]252; VBITS_EQ_512-NEXT: trn1 z2.h, z0.h, z1.h253; VBITS_EQ_512-NEXT: trn2 z0.h, z0.h, z1.h254; VBITS_EQ_512-NEXT: add z0.h, z2.h, z0.h255; VBITS_EQ_512-NEXT: str z0, [x0]256; VBITS_EQ_512-NEXT: ret257 %tmp1 = load <32 x i16>, ptr %a258 %tmp2 = load <32 x i16>, ptr %b259 %tmp3 = shufflevector <32 x i16> %tmp1, <32 x i16> %tmp2, <32 x i32> <i32 0, i32 32, i32 2, i32 34, i32 4, i32 36, i32 6, i32 38, i32 8, i32 40, i32 10, i32 42, i32 12, i32 44, i32 14, i32 46, i32 16, i32 48, i32 18, i32 50, i32 20, i32 52, i32 22, i32 54, i32 24, i32 56, i32 26, i32 58, i32 28, i32 60, i32 30, i32 62>260 %tmp4 = shufflevector <32 x i16> %tmp1, <32 x i16> %tmp2, <32 x i32> <i32 1, i32 33, i32 3, i32 35, i32 poison, i32 37, i32 7, i32 poison, i32 poison, i32 41, i32 11, i32 43, i32 13, i32 45, i32 15, i32 47, i32 17, i32 49, i32 19, i32 51, i32 21, i32 53, i32 23, i32 55, i32 25, i32 57, i32 27, i32 59, i32 29, i32 61, i32 31, i32 63>261 %tmp5 = add <32 x i16> %tmp3, %tmp4262 store <32 x i16> %tmp5, ptr %a263 ret void264}265 266define void @trn_v16i16(ptr %a, ptr %b) #0 {267; VBITS_EQ_256-LABEL: trn_v16i16:268; VBITS_EQ_256: // %bb.0:269; VBITS_EQ_256-NEXT: ldr z0, [x0]270; VBITS_EQ_256-NEXT: ldr z1, [x1]271; VBITS_EQ_256-NEXT: trn1 z2.h, z0.h, z1.h272; VBITS_EQ_256-NEXT: trn2 z0.h, z0.h, z1.h273; VBITS_EQ_256-NEXT: add z0.h, z2.h, z0.h274; VBITS_EQ_256-NEXT: str z0, [x0]275; VBITS_EQ_256-NEXT: ret276;277; VBITS_EQ_512-LABEL: trn_v16i16:278; VBITS_EQ_512: // %bb.0:279; VBITS_EQ_512-NEXT: ptrue p0.h, vl16280; VBITS_EQ_512-NEXT: ld1h { z0.h }, p0/z, [x0]281; VBITS_EQ_512-NEXT: ld1h { z1.h }, p0/z, [x1]282; VBITS_EQ_512-NEXT: trn1 z2.h, z0.h, z1.h283; VBITS_EQ_512-NEXT: trn2 z0.h, z0.h, z1.h284; VBITS_EQ_512-NEXT: add z0.h, z2.h, z0.h285; VBITS_EQ_512-NEXT: st1h { z0.h }, p0, [x0]286; VBITS_EQ_512-NEXT: ret287 %tmp1 = load <16 x i16>, ptr %a288 %tmp2 = load <16 x i16>, ptr %b289 %tmp3 = shufflevector <16 x i16> %tmp1, <16 x i16> %tmp2, <16 x i32> <i32 0, i32 16, i32 2, i32 18, i32 4, i32 20, i32 6, i32 22, i32 8, i32 24, i32 10, i32 26, i32 12, i32 28, i32 14, i32 30>290 %tmp4 = shufflevector <16 x i16> %tmp1, <16 x i16> %tmp2, <16 x i32> <i32 1, i32 17, i32 3, i32 19, i32 5, i32 21, i32 7, i32 23, i32 9, i32 25, i32 11, i32 27, i32 13, i32 29, i32 15, i32 31>291 %tmp5 = add <16 x i16> %tmp3, %tmp4292 store <16 x i16> %tmp5, ptr %a293 ret void294}295 296define void @trn_v8i32(ptr %a, ptr %b) #0 {297; VBITS_EQ_256-LABEL: trn_v8i32:298; VBITS_EQ_256: // %bb.0:299; VBITS_EQ_256-NEXT: ldr z0, [x0]300; VBITS_EQ_256-NEXT: ldr z1, [x1]301; VBITS_EQ_256-NEXT: trn1 z2.s, z0.s, z1.s302; VBITS_EQ_256-NEXT: trn2 z0.s, z0.s, z1.s303; VBITS_EQ_256-NEXT: add z0.s, z2.s, z0.s304; VBITS_EQ_256-NEXT: str z0, [x0]305; VBITS_EQ_256-NEXT: ret306;307; VBITS_EQ_512-LABEL: trn_v8i32:308; VBITS_EQ_512: // %bb.0:309; VBITS_EQ_512-NEXT: ptrue p0.s, vl8310; VBITS_EQ_512-NEXT: ld1w { z0.s }, p0/z, [x0]311; VBITS_EQ_512-NEXT: ld1w { z1.s }, p0/z, [x1]312; VBITS_EQ_512-NEXT: trn1 z2.s, z0.s, z1.s313; VBITS_EQ_512-NEXT: trn2 z0.s, z0.s, z1.s314; VBITS_EQ_512-NEXT: add z0.s, z2.s, z0.s315; VBITS_EQ_512-NEXT: st1w { z0.s }, p0, [x0]316; VBITS_EQ_512-NEXT: ret317 %tmp1 = load <8 x i32>, ptr %a318 %tmp2 = load <8 x i32>, ptr %b319 %tmp3 = shufflevector <8 x i32> %tmp1, <8 x i32> %tmp2, <8 x i32> <i32 0, i32 8, i32 poison, i32 poison, i32 4, i32 12, i32 6, i32 14>320 %tmp4 = shufflevector <8 x i32> %tmp1, <8 x i32> %tmp2, <8 x i32> <i32 1, i32 poison, i32 3, i32 11, i32 5, i32 13, i32 poison, i32 poison>321 %tmp5 = add <8 x i32> %tmp3, %tmp4322 store <8 x i32> %tmp5, ptr %a323 ret void324}325 326define void @trn_v4f64(ptr %a, ptr %b) #0 {327; VBITS_EQ_256-LABEL: trn_v4f64:328; VBITS_EQ_256: // %bb.0:329; VBITS_EQ_256-NEXT: ldr z0, [x0]330; VBITS_EQ_256-NEXT: ldr z1, [x1]331; VBITS_EQ_256-NEXT: trn1 z2.d, z0.d, z1.d332; VBITS_EQ_256-NEXT: trn2 z0.d, z0.d, z1.d333; VBITS_EQ_256-NEXT: fadd z0.d, z2.d, z0.d334; VBITS_EQ_256-NEXT: str z0, [x0]335; VBITS_EQ_256-NEXT: ret336;337; VBITS_EQ_512-LABEL: trn_v4f64:338; VBITS_EQ_512: // %bb.0:339; VBITS_EQ_512-NEXT: ptrue p0.d, vl4340; VBITS_EQ_512-NEXT: ld1d { z0.d }, p0/z, [x0]341; VBITS_EQ_512-NEXT: ld1d { z1.d }, p0/z, [x1]342; VBITS_EQ_512-NEXT: trn1 z2.d, z0.d, z1.d343; VBITS_EQ_512-NEXT: trn2 z0.d, z0.d, z1.d344; VBITS_EQ_512-NEXT: fadd z0.d, p0/m, z0.d, z2.d345; VBITS_EQ_512-NEXT: st1d { z0.d }, p0, [x0]346; VBITS_EQ_512-NEXT: ret347 %tmp1 = load <4 x double>, ptr %a348 %tmp2 = load <4 x double>, ptr %b349 %tmp3 = shufflevector <4 x double> %tmp1, <4 x double> %tmp2, <4 x i32> <i32 0, i32 4, i32 2, i32 6>350 %tmp4 = shufflevector <4 x double> %tmp1, <4 x double> %tmp2, <4 x i32> <i32 1, i32 5, i32 3, i32 7>351 %tmp5 = fadd <4 x double> %tmp3, %tmp4352 store <4 x double> %tmp5, ptr %a353 ret void354}355 356; Don't use SVE for 128-bit vectors357define void @trn_v4f32(ptr %a, ptr %b) #0 {358; CHECK-LABEL: trn_v4f32:359; CHECK: // %bb.0:360; CHECK-NEXT: ldr q0, [x0]361; CHECK-NEXT: ldr q1, [x1]362; CHECK-NEXT: trn1 v2.4s, v0.4s, v1.4s363; CHECK-NEXT: trn2 v0.4s, v0.4s, v1.4s364; CHECK-NEXT: fadd v0.4s, v2.4s, v0.4s365; CHECK-NEXT: str q0, [x0]366; CHECK-NEXT: ret367 %tmp1 = load <4 x float>, ptr %a368 %tmp2 = load <4 x float>, ptr %b369 %tmp3 = shufflevector <4 x float> %tmp1, <4 x float> %tmp2, <4 x i32> <i32 0, i32 4, i32 2, i32 6>370 %tmp4 = shufflevector <4 x float> %tmp1, <4 x float> %tmp2, <4 x i32> <i32 1, i32 5, i32 3, i32 7>371 %tmp5 = fadd <4 x float> %tmp3, %tmp4372 store <4 x float> %tmp5, ptr %a373 ret void374}375 376define void @trn_v8i32_undef(ptr %a) #0 {377; VBITS_EQ_256-LABEL: trn_v8i32_undef:378; VBITS_EQ_256: // %bb.0:379; VBITS_EQ_256-NEXT: ldr z0, [x0]380; VBITS_EQ_256-NEXT: trn1 z1.s, z0.s, z0.s381; VBITS_EQ_256-NEXT: trn2 z0.s, z0.s, z0.s382; VBITS_EQ_256-NEXT: add z0.s, z1.s, z0.s383; VBITS_EQ_256-NEXT: str z0, [x0]384; VBITS_EQ_256-NEXT: ret385;386; VBITS_EQ_512-LABEL: trn_v8i32_undef:387; VBITS_EQ_512: // %bb.0:388; VBITS_EQ_512-NEXT: ptrue p0.s, vl8389; VBITS_EQ_512-NEXT: ld1w { z0.s }, p0/z, [x0]390; VBITS_EQ_512-NEXT: trn1 z1.s, z0.s, z0.s391; VBITS_EQ_512-NEXT: trn2 z0.s, z0.s, z0.s392; VBITS_EQ_512-NEXT: add z0.s, z1.s, z0.s393; VBITS_EQ_512-NEXT: st1w { z0.s }, p0, [x0]394; VBITS_EQ_512-NEXT: ret395 %tmp1 = load <8 x i32>, ptr %a396 %tmp3 = shufflevector <8 x i32> %tmp1, <8 x i32> poison, <8 x i32> <i32 0, i32 0, i32 2, i32 2, i32 4, i32 4, i32 6, i32 6>397 %tmp4 = shufflevector <8 x i32> %tmp1, <8 x i32> poison, <8 x i32> <i32 1, i32 1, i32 3, i32 3, i32 5, i32 5, i32 7, i32 7>398 %tmp5 = add <8 x i32> %tmp3, %tmp4399 store <8 x i32> %tmp5, ptr %a400 ret void401}402 403; Emit zip2 instruction for v32i8 shuffle with vscale_range(2,2),404; since the size of v32i8 is the same as the runtime vector length.405define void @zip2_v32i8(ptr %a, ptr %b) #1 {406; CHECK-LABEL: zip2_v32i8:407; CHECK: // %bb.0:408; CHECK-NEXT: ldr z0, [x0]409; CHECK-NEXT: ldr z1, [x1]410; CHECK-NEXT: zip2 z0.b, z0.b, z1.b411; CHECK-NEXT: str z0, [x0]412; CHECK-NEXT: ret413 %tmp1 = load volatile <32 x i8>, ptr %a414 %tmp2 = load volatile <32 x i8>, ptr %b415 %tmp3 = shufflevector <32 x i8> %tmp1, <32 x i8> %tmp2, <32 x i32> <i32 16, i32 48, i32 17, i32 49, i32 18, i32 50, i32 19, i32 51, i32 20, i32 52, i32 21, i32 53, i32 22, i32 54, i32 23, i32 55, i32 24, i32 56, i32 25, i32 57, i32 26, i32 58, i32 27, i32 59, i32 28, i32 60, i32 29, i32 61, i32 30, i32 62, i32 31, i32 63>416 store volatile <32 x i8> %tmp3, ptr %a417 ret void418}419 420; Emit zip2 instruction for v16i16 shuffle with vscale_range(2,2),421; since the size of v16i16 is the same as the runtime vector length.422define void @zip2_v16i16(ptr %a, ptr %b) #1 {423; CHECK-LABEL: zip2_v16i16:424; CHECK: // %bb.0:425; CHECK-NEXT: ldr z0, [x0]426; CHECK-NEXT: ldr z1, [x1]427; CHECK-NEXT: zip2 z0.h, z0.h, z1.h428; CHECK-NEXT: str z0, [x0]429; CHECK-NEXT: ret430 %tmp1 = load volatile <16 x i16>, ptr %a431 %tmp2 = load volatile <16 x i16>, ptr %b432 %tmp3 = shufflevector <16 x i16> %tmp1, <16 x i16> %tmp2, <16 x i32> <i32 8, i32 24, i32 9, i32 25, i32 10, i32 26, i32 11, i32 27, i32 12, i32 28, i32 13, i32 29, i32 14, i32 30, i32 15, i32 31>433 store volatile <16 x i16> %tmp3, ptr %a434 ret void435}436 437; Emit zip2 instruction for v8i32 shuffle with vscale_range(2,2),438; since the size of v8i32 is the same as the runtime vector length.439define void @zip2_v8i32(ptr %a, ptr %b) #1 {440; CHECK-LABEL: zip2_v8i32:441; CHECK: // %bb.0:442; CHECK-NEXT: ldr z0, [x0]443; CHECK-NEXT: ldr z1, [x1]444; CHECK-NEXT: zip2 z0.s, z0.s, z1.s445; CHECK-NEXT: str z0, [x0]446; CHECK-NEXT: ret447 %tmp1 = load volatile <8 x i32>, ptr %a448 %tmp2 = load volatile <8 x i32>, ptr %b449 %tmp3 = shufflevector <8 x i32> %tmp1, <8 x i32> %tmp2, <8 x i32> <i32 4, i32 12, i32 5, i32 13, i32 6, i32 14, i32 7, i32 15>450 store volatile <8 x i32> %tmp3, ptr %a451 ret void452}453 454; Emit zip2 instruction for v8i32 and undef shuffle with vscale_range(2,2)455define void @zip2_v8i32_undef(ptr %a) #1 {456; CHECK-LABEL: zip2_v8i32_undef:457; CHECK: // %bb.0:458; CHECK-NEXT: ldr z0, [x0]459; CHECK-NEXT: zip2 z0.s, z0.s, z0.s460; CHECK-NEXT: str z0, [x0]461; CHECK-NEXT: ret462 %tmp1 = load volatile <8 x i32>, ptr %a463 %tmp2 = shufflevector <8 x i32> %tmp1, <8 x i32> poison, <8 x i32> <i32 4, i32 4, i32 5, i32 5, i32 6, i32 6, i32 7, i32 7>464 store volatile <8 x i32> %tmp2, ptr %a465 ret void466}467 468; Emit uzp1/2 instruction for v32i8 shuffle with vscale_range(2,2),469; since the size of v32i8 is the same as the runtime vector length.470define void @uzp_v32i8(ptr %a, ptr %b) #1 {471; CHECK-LABEL: uzp_v32i8:472; CHECK: // %bb.0:473; CHECK-NEXT: ldr z0, [x0]474; CHECK-NEXT: ldr z1, [x1]475; CHECK-NEXT: uzp1 z2.b, z0.b, z1.b476; CHECK-NEXT: uzp2 z0.b, z0.b, z1.b477; CHECK-NEXT: add z0.b, z2.b, z0.b478; CHECK-NEXT: str z0, [x0]479; CHECK-NEXT: ret480 %tmp1 = load <32 x i8>, ptr %a481 %tmp2 = load <32 x i8>, ptr %b482 %tmp3 = shufflevector <32 x i8> %tmp1, <32 x i8> %tmp2, <32 x i32> <i32 0, i32 2, i32 4, i32 6, i32 8, i32 10, i32 12, i32 14, i32 16, i32 18, i32 20, i32 22, i32 24, i32 26, i32 28, i32 30, i32 32, i32 34, i32 36, i32 38, i32 40, i32 42, i32 44, i32 46, i32 48, i32 50, i32 52, i32 54, i32 56, i32 58, i32 60, i32 62>483 %tmp4 = shufflevector <32 x i8> %tmp1, <32 x i8> %tmp2, <32 x i32> <i32 1, i32 3, i32 5, i32 poison, i32 9, i32 11, i32 13, i32 poison, i32 poison, i32 19, i32 21, i32 23, i32 25, i32 27, i32 29, i32 31, i32 33, i32 35, i32 37, i32 39, i32 41, i32 43, i32 45, i32 47, i32 49, i32 51, i32 53, i32 55, i32 57, i32 59, i32 61, i32 63>484 %tmp5 = add <32 x i8> %tmp3, %tmp4485 store <32 x i8> %tmp5, ptr %a486 ret void487}488 489; Emit uzp1/2 instruction for v32i16 shuffle with vscale_range(2,2),490; v32i16 will be expanded into two v16i16, and the size of v16i16 is491; the same as the runtime vector length.492define void @uzp_v32i16(ptr %a, ptr %b) #1 {493; CHECK-LABEL: uzp_v32i16:494; CHECK: // %bb.0:495; CHECK-NEXT: ldr z0, [x0, #1, mul vl]496; CHECK-NEXT: ldr z1, [x0]497; CHECK-NEXT: ldr z2, [x1, #1, mul vl]498; CHECK-NEXT: ldr z3, [x1]499; CHECK-NEXT: uzp1 z4.h, z1.h, z0.h500; CHECK-NEXT: uzp2 z0.h, z1.h, z0.h501; CHECK-NEXT: uzp1 z1.h, z3.h, z2.h502; CHECK-NEXT: uzp2 z2.h, z3.h, z2.h503; CHECK-NEXT: add z0.h, z4.h, z0.h504; CHECK-NEXT: add z1.h, z1.h, z2.h505; CHECK-NEXT: str z0, [x0]506; CHECK-NEXT: str z1, [x0, #1, mul vl]507; CHECK-NEXT: ret508 %tmp1 = load <32 x i16>, ptr %a509 %tmp2 = load <32 x i16>, ptr %b510 %tmp3 = shufflevector <32 x i16> %tmp1, <32 x i16> %tmp2, <32 x i32> <i32 0, i32 2, i32 4, i32 6, i32 8, i32 10, i32 12, i32 14, i32 16, i32 18, i32 20, i32 22, i32 24, i32 26, i32 28, i32 30, i32 32, i32 34, i32 36, i32 38, i32 40, i32 42, i32 44, i32 46, i32 48, i32 50, i32 52, i32 54, i32 56, i32 58, i32 60, i32 62>511 %tmp4 = shufflevector <32 x i16> %tmp1, <32 x i16> %tmp2, <32 x i32> <i32 1, i32 3, i32 5, i32 poison, i32 9, i32 11, i32 13, i32 poison, i32 poison, i32 19, i32 21, i32 23, i32 25, i32 27, i32 29, i32 31, i32 33, i32 35, i32 37, i32 39, i32 41, i32 43, i32 45, i32 47, i32 49, i32 51, i32 53, i32 55, i32 57, i32 59, i32 61, i32 63>512 %tmp5 = add <32 x i16> %tmp3, %tmp4513 store <32 x i16> %tmp5, ptr %a514 ret void515}516 517; Emit uzp1/2 instruction for v16i16 shuffle with vscale_range(2,2),518; since the size of v16i16 is the same as the runtime vector length.519define void @uzp_v16i16(ptr %a, ptr %b) #1 {520; CHECK-LABEL: uzp_v16i16:521; CHECK: // %bb.0:522; CHECK-NEXT: ldr z0, [x0]523; CHECK-NEXT: ldr z1, [x1]524; CHECK-NEXT: uzp1 z2.h, z0.h, z1.h525; CHECK-NEXT: uzp2 z0.h, z0.h, z1.h526; CHECK-NEXT: add z0.h, z2.h, z0.h527; CHECK-NEXT: str z0, [x0]528; CHECK-NEXT: ret529 %tmp1 = load <16 x i16>, ptr %a530 %tmp2 = load <16 x i16>, ptr %b531 %tmp3 = shufflevector <16 x i16> %tmp1, <16 x i16> %tmp2, <16 x i32> <i32 0, i32 2, i32 4, i32 6, i32 8, i32 10, i32 12, i32 14, i32 16, i32 18, i32 20, i32 22, i32 24, i32 26, i32 28, i32 30>532 %tmp4 = shufflevector <16 x i16> %tmp1, <16 x i16> %tmp2, <16 x i32> <i32 1, i32 3, i32 5, i32 7, i32 9, i32 11, i32 13, i32 15, i32 17, i32 19, i32 21, i32 23, i32 25, i32 27, i32 29, i32 31>533 %tmp5 = add <16 x i16> %tmp3, %tmp4534 store <16 x i16> %tmp5, ptr %a535 ret void536}537 538; Emit uzp1/2 instruction for v8f32 shuffle with vscale_range(2,2),539; since the size of v8f32 is the same as the runtime vector length.540define void @uzp_v8f32(ptr %a, ptr %b) #1 {541; CHECK-LABEL: uzp_v8f32:542; CHECK: // %bb.0:543; CHECK-NEXT: ldr z0, [x0]544; CHECK-NEXT: ldr z1, [x1]545; CHECK-NEXT: uzp1 z2.s, z0.s, z1.s546; CHECK-NEXT: uzp2 z0.s, z0.s, z1.s547; CHECK-NEXT: fadd z0.s, z2.s, z0.s548; CHECK-NEXT: str z0, [x0]549; CHECK-NEXT: ret550 %tmp1 = load <8 x float>, ptr %a551 %tmp2 = load <8 x float>, ptr %b552 %tmp3 = shufflevector <8 x float> %tmp1, <8 x float> %tmp2, <8 x i32> <i32 0, i32 poison, i32 4, i32 6, i32 poison, i32 10, i32 12, i32 14>553 %tmp4 = shufflevector <8 x float> %tmp1, <8 x float> %tmp2, <8 x i32> <i32 1, i32 poison, i32 5, i32 7, i32 9, i32 11, i32 poison, i32 poison>554 %tmp5 = fadd <8 x float> %tmp3, %tmp4555 store <8 x float> %tmp5, ptr %a556 ret void557}558 559; Emit uzp1/2 instruction for v4i64 shuffle with vscale_range(2,2),560; since the size of v4i64 is the same as the runtime vector length.561define void @uzp_v4i64(ptr %a, ptr %b) #1 {562; CHECK-LABEL: uzp_v4i64:563; CHECK: // %bb.0:564; CHECK-NEXT: ldr z0, [x0]565; CHECK-NEXT: ldr z1, [x1]566; CHECK-NEXT: uzp1 z2.d, z0.d, z1.d567; CHECK-NEXT: uzp2 z0.d, z0.d, z1.d568; CHECK-NEXT: add z0.d, z2.d, z0.d569; CHECK-NEXT: str z0, [x0]570; CHECK-NEXT: ret571 %tmp1 = load <4 x i64>, ptr %a572 %tmp2 = load <4 x i64>, ptr %b573 %tmp3 = shufflevector <4 x i64> %tmp1, <4 x i64> %tmp2, <4 x i32> <i32 0, i32 2, i32 4, i32 6>574 %tmp4 = shufflevector <4 x i64> %tmp1, <4 x i64> %tmp2, <4 x i32> <i32 1, i32 3, i32 5, i32 7>575 %tmp5 = add <4 x i64> %tmp3, %tmp4576 store <4 x i64> %tmp5, ptr %a577 ret void578}579 580; Don't use SVE for 128-bit vectors581define void @uzp_v8i16(ptr %a, ptr %b) #1 {582; CHECK-LABEL: uzp_v8i16:583; CHECK: // %bb.0:584; CHECK-NEXT: ldr q0, [x0]585; CHECK-NEXT: ldr q1, [x1]586; CHECK-NEXT: uzp1 v2.8h, v0.8h, v1.8h587; CHECK-NEXT: uzp2 v0.8h, v0.8h, v1.8h588; CHECK-NEXT: eor v0.16b, v2.16b, v0.16b589; CHECK-NEXT: str q0, [x0]590; CHECK-NEXT: ret591 %tmp1 = load <8 x i16>, ptr %a592 %tmp2 = load <8 x i16>, ptr %b593 %tmp3 = shufflevector <8 x i16> %tmp1, <8 x i16> %tmp2, <8 x i32> <i32 0, i32 2, i32 4, i32 6, i32 8, i32 10, i32 12, i32 14>594 %tmp4 = shufflevector <8 x i16> %tmp1, <8 x i16> %tmp2, <8 x i32> <i32 1, i32 3, i32 5, i32 7, i32 9, i32 11, i32 13, i32 15>595 %tmp5 = xor <8 x i16> %tmp3, %tmp4596 store <8 x i16> %tmp5, ptr %a597 ret void598}599 600; Emit uzp1/2 instruction for v8i32 and undef shuffle with vscale_range(2,2)601define void @uzp_v8i32_undef(ptr %a) #1 {602; CHECK-LABEL: uzp_v8i32_undef:603; CHECK: // %bb.0:604; CHECK-NEXT: ldr z0, [x0]605; CHECK-NEXT: uzp1 z1.s, z0.s, z0.s606; CHECK-NEXT: uzp2 z0.s, z0.s, z0.s607; CHECK-NEXT: add z0.s, z1.s, z0.s608; CHECK-NEXT: str z0, [x0]609; CHECK-NEXT: ret610 %tmp1 = load <8 x i32>, ptr %a611 %tmp3 = shufflevector <8 x i32> %tmp1, <8 x i32> poison, <8 x i32> <i32 0, i32 2, i32 4, i32 6, i32 0, i32 2, i32 4, i32 6>612 %tmp4 = shufflevector <8 x i32> %tmp1, <8 x i32> poison, <8 x i32> <i32 1, i32 3, i32 5, i32 7, i32 1, i32 3, i32 5, i32 7>613 %tmp5 = add <8 x i32> %tmp3, %tmp4614 store <8 x i32> %tmp5, ptr %a615 ret void616}617 618; Only zip1 can be emitted safely with vscale_range(2,4).619; vscale_range(2,4) means different min/max vector sizes, zip2 relies on620; knowing which indices represent the high half of sve vector register.621define void @zip_vscale2_4(ptr %a, ptr %b) #2 {622; CHECK-LABEL: zip_vscale2_4:623; CHECK: // %bb.0:624; CHECK-NEXT: stp x29, x30, [sp, #-16]! // 16-byte Folded Spill625; CHECK-NEXT: sub x9, sp, #48626; CHECK-NEXT: mov x29, sp627; CHECK-NEXT: and sp, x9, #0xffffffffffffffe0628; CHECK-NEXT: .cfi_def_cfa w29, 16629; CHECK-NEXT: .cfi_offset w30, -8630; CHECK-NEXT: .cfi_offset w29, -16631; CHECK-NEXT: ptrue p0.d, vl4632; CHECK-NEXT: mov x8, sp633; CHECK-NEXT: ld1d { z0.d }, p0/z, [x1]634; CHECK-NEXT: ld1d { z1.d }, p0/z, [x0]635; CHECK-NEXT: mov z2.d, z0.d[3]636; CHECK-NEXT: mov z3.d, z1.d[3]637; CHECK-NEXT: mov z4.d, z0.d[2]638; CHECK-NEXT: mov z5.d, z1.d[2]639; CHECK-NEXT: zip1 z0.d, z1.d, z0.d640; CHECK-NEXT: stp d3, d2, [sp, #16]641; CHECK-NEXT: stp d5, d4, [sp]642; CHECK-NEXT: ld1d { z1.d }, p0/z, [x8]643; CHECK-NEXT: fadd z0.d, p0/m, z0.d, z1.d644; CHECK-NEXT: st1d { z0.d }, p0, [x0]645; CHECK-NEXT: mov sp, x29646; CHECK-NEXT: ldp x29, x30, [sp], #16 // 16-byte Folded Reload647; CHECK-NEXT: ret648 %tmp1 = load <4 x double>, ptr %a649 %tmp2 = load <4 x double>, ptr %b650 %tmp3 = shufflevector <4 x double> %tmp1, <4 x double> %tmp2, <4 x i32> <i32 0, i32 4, i32 1, i32 5>651 %tmp4 = shufflevector <4 x double> %tmp1, <4 x double> %tmp2, <4 x i32> <i32 2, i32 6, i32 3, i32 7>652 %tmp5 = fadd <4 x double> %tmp3, %tmp4653 store <4 x double> %tmp5, ptr %a654 ret void655}656 657attributes #0 = { "target-features"="+sve" }658attributes #1 = { "target-features"="+sve" vscale_range(2,2) }659attributes #2 = { "target-features"="+sve" vscale_range(2,4) }660