3290 lines · plain
1; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py2; RUN: llc -mtriple=arm64-apple-ios -mattr=+sve -o - %s | FileCheck %s3; RUN: llc -mtriple=aarch64_be-unknown-linux -mattr=+sve -o - %s | FileCheck --check-prefix=CHECK-BE %s4 5; CHECK-LABEL: lCPI0_0:6; CHECK-NEXT: .byte 0 ; 0x07; CHECK-NEXT: .byte 255 ; 0xff8; CHECK-NEXT: .byte 255 ; 0xff9; CHECK-NEXT: .byte 255 ; 0xff10; CHECK-NEXT: .byte 1 ; 0x111; CHECK-NEXT: .byte 255 ; 0xff12; CHECK-NEXT: .byte 255 ; 0xff13; CHECK-NEXT: .byte 255 ; 0xff14; CHECK-NEXT: .byte 2 ; 0x215; CHECK-NEXT: .byte 255 ; 0xff16; CHECK-NEXT: .byte 255 ; 0xff17; CHECK-NEXT: .byte 255 ; 0xff18; CHECK-NEXT: .byte 3 ; 0x319; CHECK-NEXT: .byte 255 ; 0xff20; CHECK-NEXT: .byte 255 ; 0xff21; CHECK-NEXT: .byte 255 ; 0xff22; CHECK-NEXT:lCPI0_1:23; CHECK-NEXT: .byte 4 ; 0x424; CHECK-NEXT: .byte 255 ; 0xff25; CHECK-NEXT: .byte 255 ; 0xff26; CHECK-NEXT: .byte 255 ; 0xff27; CHECK-NEXT: .byte 5 ; 0x528; CHECK-NEXT: .byte 255 ; 0xff29; CHECK-NEXT: .byte 255 ; 0xff30; CHECK-NEXT: .byte 255 ; 0xff31; CHECK-NEXT: .byte 6 ; 0x632; CHECK-NEXT: .byte 255 ; 0xff33; CHECK-NEXT: .byte 255 ; 0xff34; CHECK-NEXT: .byte 255 ; 0xff35; CHECK-NEXT: .byte 7 ; 0x736; CHECK-NEXT: .byte 255 ; 0xff37; CHECK-NEXT: .byte 255 ; 0xff38; CHECK-NEXT: .byte 255 ; 0xff39; CHECK-NEXT:lCPI0_2:40; CHECK-NEXT: .byte 8 ; 0x841; CHECK-NEXT: .byte 255 ; 0xff42; CHECK-NEXT: .byte 255 ; 0xff43; CHECK-NEXT: .byte 255 ; 0xff44; CHECK-NEXT: .byte 9 ; 0x945; CHECK-NEXT: .byte 255 ; 0xff46; CHECK-NEXT: .byte 255 ; 0xff47; CHECK-NEXT: .byte 255 ; 0xff48; CHECK-NEXT: .byte 10 ; 0xa49; CHECK-NEXT: .byte 255 ; 0xff50; CHECK-NEXT: .byte 255 ; 0xff51; CHECK-NEXT: .byte 255 ; 0xff52; CHECK-NEXT: .byte 11 ; 0xb53; CHECK-NEXT: .byte 255 ; 0xff54; CHECK-NEXT: .byte 255 ; 0xff55; CHECK-NEXT: .byte 255 ; 0xff56; CHECK-NEXT:lCPI0_3:57; CHECK-NEXT: .byte 12 ; 0xc58; CHECK-NEXT: .byte 255 ; 0xff59; CHECK-NEXT: .byte 255 ; 0xff60; CHECK-NEXT: .byte 255 ; 0xff61; CHECK-NEXT: .byte 13 ; 0xd62; CHECK-NEXT: .byte 255 ; 0xff63; CHECK-NEXT: .byte 255 ; 0xff64; CHECK-NEXT: .byte 255 ; 0xff65; CHECK-NEXT: .byte 14 ; 0xe66; CHECK-NEXT: .byte 255 ; 0xff67; CHECK-NEXT: .byte 255 ; 0xff68; CHECK-NEXT: .byte 255 ; 0xff69; CHECK-NEXT: .byte 15 ; 0xf70; CHECK-NEXT: .byte 255 ; 0xff71; CHECK-NEXT: .byte 255 ; 0xff72; CHECK-NEXT: .byte 255 ; 0xff73 74; CHECK-BE: .LCPI0_0:75; CHECK-BE-NEXT: .byte 255 // 0xff76; CHECK-BE-NEXT: .byte 255 // 0xff77; CHECK-BE-NEXT: .byte 255 // 0xff78; CHECK-BE-NEXT: .byte 0 // 0x079; CHECK-BE-NEXT: .byte 255 // 0xff80; CHECK-BE-NEXT: .byte 255 // 0xff81; CHECK-BE-NEXT: .byte 255 // 0xff82; CHECK-BE-NEXT: .byte 1 // 0x183; CHECK-BE-NEXT: .byte 255 // 0xff84; CHECK-BE-NEXT: .byte 255 // 0xff85; CHECK-BE-NEXT: .byte 255 // 0xff86; CHECK-BE-NEXT: .byte 2 // 0x287; CHECK-BE-NEXT: .byte 255 // 0xff88; CHECK-BE-NEXT: .byte 255 // 0xff89; CHECK-BE-NEXT: .byte 255 // 0xff90; CHECK-BE-NEXT: .byte 3 // 0x391; CHECK-BE-NEXT: .LCPI0_1:92; CHECK-BE-NEXT: .byte 255 // 0xff93; CHECK-BE-NEXT: .byte 255 // 0xff94; CHECK-BE-NEXT: .byte 255 // 0xff95; CHECK-BE-NEXT: .byte 4 // 0x496; CHECK-BE-NEXT: .byte 255 // 0xff97; CHECK-BE-NEXT: .byte 255 // 0xff98; CHECK-BE-NEXT: .byte 255 // 0xff99; CHECK-BE-NEXT: .byte 5 // 0x5100; CHECK-BE-NEXT: .byte 255 // 0xff101; CHECK-BE-NEXT: .byte 255 // 0xff102; CHECK-BE-NEXT: .byte 255 // 0xff103; CHECK-BE-NEXT: .byte 6 // 0x6104; CHECK-BE-NEXT: .byte 255 // 0xff105; CHECK-BE-NEXT: .byte 255 // 0xff106; CHECK-BE-NEXT: .byte 255 // 0xff107; CHECK-BE-NEXT: .byte 7 // 0x7108; CHECK-BE-NEXT: .LCPI0_2:109; CHECK-BE-NEXT: .byte 255 // 0xff110; CHECK-BE-NEXT: .byte 255 // 0xff111; CHECK-BE-NEXT: .byte 255 // 0xff112; CHECK-BE-NEXT: .byte 8 // 0x8113; CHECK-BE-NEXT: .byte 255 // 0xff114; CHECK-BE-NEXT: .byte 255 // 0xff115; CHECK-BE-NEXT: .byte 255 // 0xff116; CHECK-BE-NEXT: .byte 9 // 0x9117; CHECK-BE-NEXT: .byte 255 // 0xff118; CHECK-BE-NEXT: .byte 255 // 0xff119; CHECK-BE-NEXT: .byte 255 // 0xff120; CHECK-BE-NEXT: .byte 10 // 0xa121; CHECK-BE-NEXT: .byte 255 // 0xff122; CHECK-BE-NEXT: .byte 255 // 0xff123; CHECK-BE-NEXT: .byte 255 // 0xff124; CHECK-BE-NEXT: .byte 11 // 0xb125; CHECK-BE-NEXT: .LCPI0_3:126; CHECK-BE-NEXT: .byte 255 // 0xff127; CHECK-BE-NEXT: .byte 255 // 0xff128; CHECK-BE-NEXT: .byte 255 // 0xff129; CHECK-BE-NEXT: .byte 12 // 0xc130; CHECK-BE-NEXT: .byte 255 // 0xff131; CHECK-BE-NEXT: .byte 255 // 0xff132; CHECK-BE-NEXT: .byte 255 // 0xff133; CHECK-BE-NEXT: .byte 13 // 0xd134; CHECK-BE-NEXT: .byte 255 // 0xff135; CHECK-BE-NEXT: .byte 255 // 0xff136; CHECK-BE-NEXT: .byte 255 // 0xff137; CHECK-BE-NEXT: .byte 14 // 0xe138; CHECK-BE-NEXT: .byte 255 // 0xff139; CHECK-BE-NEXT: .byte 255 // 0xff140; CHECK-BE-NEXT: .byte 255 // 0xff141; CHECK-BE-NEXT: .byte 15 // 0xf142 143; It's profitable to convert the zext to a shuffle, which in turn will be144; lowered to 4 tbl instructions. The masks are materialized outside the loop.145define void @zext_v16i8_to_v16i32_in_loop(ptr %src, ptr %dst) {146; CHECK-LABEL: zext_v16i8_to_v16i32_in_loop:147; CHECK: ; %bb.0: ; %entry148; CHECK-NEXT: Lloh0:149; CHECK-NEXT: adrp x8, lCPI0_0@PAGE150; CHECK-NEXT: Lloh1:151; CHECK-NEXT: adrp x9, lCPI0_1@PAGE152; CHECK-NEXT: Lloh2:153; CHECK-NEXT: adrp x10, lCPI0_2@PAGE154; CHECK-NEXT: Lloh3:155; CHECK-NEXT: ldr q0, [x8, lCPI0_0@PAGEOFF]156; CHECK-NEXT: Lloh4:157; CHECK-NEXT: adrp x8, lCPI0_3@PAGE158; CHECK-NEXT: Lloh5:159; CHECK-NEXT: ldr q1, [x9, lCPI0_1@PAGEOFF]160; CHECK-NEXT: Lloh6:161; CHECK-NEXT: ldr q2, [x10, lCPI0_2@PAGEOFF]162; CHECK-NEXT: Lloh7:163; CHECK-NEXT: ldr q3, [x8, lCPI0_3@PAGEOFF]164; CHECK-NEXT: mov x8, xzr165; CHECK-NEXT: LBB0_1: ; %loop166; CHECK-NEXT: ; =>This Inner Loop Header: Depth=1167; CHECK-NEXT: ldr q4, [x0, x8]168; CHECK-NEXT: add x8, x8, #16169; CHECK-NEXT: cmp x8, #128170; CHECK-NEXT: tbl.16b v5, { v4 }, v3171; CHECK-NEXT: tbl.16b v6, { v4 }, v2172; CHECK-NEXT: tbl.16b v7, { v4 }, v1173; CHECK-NEXT: tbl.16b v4, { v4 }, v0174; CHECK-NEXT: stp q6, q5, [x1, #32]175; CHECK-NEXT: stp q4, q7, [x1], #64176; CHECK-NEXT: b.ne LBB0_1177; CHECK-NEXT: ; %bb.2: ; %exit178; CHECK-NEXT: ret179; CHECK-NEXT: .loh AdrpLdr Lloh4, Lloh7180; CHECK-NEXT: .loh AdrpLdr Lloh2, Lloh6181; CHECK-NEXT: .loh AdrpLdr Lloh1, Lloh5182; CHECK-NEXT: .loh AdrpAdrp Lloh0, Lloh4183; CHECK-NEXT: .loh AdrpLdr Lloh0, Lloh3184;185; CHECK-BE-LABEL: zext_v16i8_to_v16i32_in_loop:186; CHECK-BE: // %bb.0: // %entry187; CHECK-BE-NEXT: adrp x8, .LCPI0_0188; CHECK-BE-NEXT: add x8, x8, :lo12:.LCPI0_0189; CHECK-BE-NEXT: ld1 { v0.16b }, [x8]190; CHECK-BE-NEXT: adrp x8, .LCPI0_1191; CHECK-BE-NEXT: add x8, x8, :lo12:.LCPI0_1192; CHECK-BE-NEXT: ld1 { v1.16b }, [x8]193; CHECK-BE-NEXT: adrp x8, .LCPI0_2194; CHECK-BE-NEXT: add x8, x8, :lo12:.LCPI0_2195; CHECK-BE-NEXT: ld1 { v2.16b }, [x8]196; CHECK-BE-NEXT: adrp x8, .LCPI0_3197; CHECK-BE-NEXT: add x8, x8, :lo12:.LCPI0_3198; CHECK-BE-NEXT: ld1 { v3.16b }, [x8]199; CHECK-BE-NEXT: mov x8, xzr200; CHECK-BE-NEXT: .LBB0_1: // %loop201; CHECK-BE-NEXT: // =>This Inner Loop Header: Depth=1202; CHECK-BE-NEXT: add x9, x0, x8203; CHECK-BE-NEXT: add x8, x8, #16204; CHECK-BE-NEXT: ld1 { v4.16b }, [x9]205; CHECK-BE-NEXT: add x9, x1, #48206; CHECK-BE-NEXT: cmp x8, #128207; CHECK-BE-NEXT: tbl v5.16b, { v4.16b }, v3.16b208; CHECK-BE-NEXT: tbl v6.16b, { v4.16b }, v2.16b209; CHECK-BE-NEXT: tbl v7.16b, { v4.16b }, v1.16b210; CHECK-BE-NEXT: tbl v4.16b, { v4.16b }, v0.16b211; CHECK-BE-NEXT: st1 { v5.16b }, [x9]212; CHECK-BE-NEXT: add x9, x1, #32213; CHECK-BE-NEXT: st1 { v6.16b }, [x9]214; CHECK-BE-NEXT: add x9, x1, #16215; CHECK-BE-NEXT: st1 { v4.16b }, [x1]216; CHECK-BE-NEXT: add x1, x1, #64217; CHECK-BE-NEXT: st1 { v7.16b }, [x9]218; CHECK-BE-NEXT: b.ne .LBB0_1219; CHECK-BE-NEXT: // %bb.2: // %exit220; CHECK-BE-NEXT: ret221entry:222 br label %loop223 224loop:225 %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]226 %src.gep = getelementptr i8, ptr %src, i64 %iv227 %load = load <16 x i8>, ptr %src.gep228 %ext = zext <16 x i8> %load to <16 x i32>229 %dst.gep = getelementptr i32, ptr %dst, i64 %iv230 store <16 x i32> %ext, ptr %dst.gep231 %iv.next = add nuw i64 %iv, 16232 %ec = icmp eq i64 %iv.next, 128233 br i1 %ec, label %exit, label %loop234 235exit:236 ret void237}238 239define void @zext_v16i8_to_v16i32_in_loop_not_header(ptr %src, ptr %dst, i1 %c) {240; CHECK-LABEL: zext_v16i8_to_v16i32_in_loop_not_header:241; CHECK: ; %bb.0: ; %entry242; CHECK-NEXT: mov x8, xzr243; CHECK-NEXT: b LBB1_2244; CHECK-NEXT: LBB1_1: ; %loop.latch245; CHECK-NEXT: ; in Loop: Header=BB1_2 Depth=1246; CHECK-NEXT: add x8, x8, #16247; CHECK-NEXT: add x1, x1, #64248; CHECK-NEXT: cmp x8, #128249; CHECK-NEXT: b.eq LBB1_4250; CHECK-NEXT: LBB1_2: ; %loop251; CHECK-NEXT: ; =>This Inner Loop Header: Depth=1252; CHECK-NEXT: tbz w2, #0, LBB1_1253; CHECK-NEXT: ; %bb.3: ; %then254; CHECK-NEXT: ; in Loop: Header=BB1_2 Depth=1255; CHECK-NEXT: ldr q0, [x0, x8]256; CHECK-NEXT: ushll2.8h v1, v0, #0257; CHECK-NEXT: ushll.8h v0, v0, #0258; CHECK-NEXT: ushll2.4s v2, v1, #0259; CHECK-NEXT: ushll.4s v1, v1, #0260; CHECK-NEXT: ushll2.4s v3, v0, #0261; CHECK-NEXT: ushll.4s v0, v0, #0262; CHECK-NEXT: stp q1, q2, [x1, #32]263; CHECK-NEXT: stp q0, q3, [x1]264; CHECK-NEXT: b LBB1_1265; CHECK-NEXT: LBB1_4: ; %exit266; CHECK-NEXT: ret267;268; CHECK-BE-LABEL: zext_v16i8_to_v16i32_in_loop_not_header:269; CHECK-BE: // %bb.0: // %entry270; CHECK-BE-NEXT: mov x8, xzr271; CHECK-BE-NEXT: b .LBB1_2272; CHECK-BE-NEXT: .LBB1_1: // %loop.latch273; CHECK-BE-NEXT: // in Loop: Header=BB1_2 Depth=1274; CHECK-BE-NEXT: add x8, x8, #16275; CHECK-BE-NEXT: add x1, x1, #64276; CHECK-BE-NEXT: cmp x8, #128277; CHECK-BE-NEXT: b.eq .LBB1_4278; CHECK-BE-NEXT: .LBB1_2: // %loop279; CHECK-BE-NEXT: // =>This Inner Loop Header: Depth=1280; CHECK-BE-NEXT: tbz w2, #0, .LBB1_1281; CHECK-BE-NEXT: // %bb.3: // %then282; CHECK-BE-NEXT: // in Loop: Header=BB1_2 Depth=1283; CHECK-BE-NEXT: add x9, x0, x8284; CHECK-BE-NEXT: add x10, x1, #32285; CHECK-BE-NEXT: ld1 { v0.16b }, [x9]286; CHECK-BE-NEXT: add x9, x1, #48287; CHECK-BE-NEXT: ushll2 v1.8h, v0.16b, #0288; CHECK-BE-NEXT: ushll v0.8h, v0.8b, #0289; CHECK-BE-NEXT: ushll2 v2.4s, v1.8h, #0290; CHECK-BE-NEXT: ushll v1.4s, v1.4h, #0291; CHECK-BE-NEXT: ushll2 v3.4s, v0.8h, #0292; CHECK-BE-NEXT: ushll v0.4s, v0.4h, #0293; CHECK-BE-NEXT: st1 { v2.4s }, [x9]294; CHECK-BE-NEXT: add x9, x1, #16295; CHECK-BE-NEXT: st1 { v1.4s }, [x10]296; CHECK-BE-NEXT: st1 { v3.4s }, [x9]297; CHECK-BE-NEXT: st1 { v0.4s }, [x1]298; CHECK-BE-NEXT: b .LBB1_1299; CHECK-BE-NEXT: .LBB1_4: // %exit300; CHECK-BE-NEXT: ret301entry:302 br label %loop303 304loop:305 %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop.latch ]306 br i1 %c, label %then, label %loop.latch307 308then:309 %src.gep = getelementptr i8, ptr %src, i64 %iv310 %load = load <16 x i8>, ptr %src.gep311 %ext = zext <16 x i8> %load to <16 x i32>312 %dst.gep = getelementptr i32, ptr %dst, i64 %iv313 store <16 x i32> %ext, ptr %dst.gep314 br label %loop.latch315 316loop.latch:317 %iv.next = add nuw i64 %iv, 16318 %ec = icmp eq i64 %iv.next, 128319 br i1 %ec, label %exit, label %loop320 321exit:322 ret void323}324 325; Not profitable to use shuffle/tbl, as 4 tbls + materializing the masks326; require more instructions than lowering zext directly.327define void @zext_v16i8_to_v16i32_no_loop(ptr %src, ptr %dst) {328; CHECK-LABEL: zext_v16i8_to_v16i32_no_loop:329; CHECK: ; %bb.0: ; %entry330; CHECK-NEXT: ldr q0, [x0]331; CHECK-NEXT: ushll2.8h v1, v0, #0332; CHECK-NEXT: ushll.8h v0, v0, #0333; CHECK-NEXT: ushll2.4s v2, v1, #0334; CHECK-NEXT: ushll.4s v1, v1, #0335; CHECK-NEXT: ushll2.4s v3, v0, #0336; CHECK-NEXT: ushll.4s v0, v0, #0337; CHECK-NEXT: stp q1, q2, [x1, #32]338; CHECK-NEXT: stp q0, q3, [x1]339; CHECK-NEXT: ret340;341; CHECK-BE-LABEL: zext_v16i8_to_v16i32_no_loop:342; CHECK-BE: // %bb.0: // %entry343; CHECK-BE-NEXT: ld1 { v0.16b }, [x0]344; CHECK-BE-NEXT: add x8, x1, #48345; CHECK-BE-NEXT: ushll2 v1.8h, v0.16b, #0346; CHECK-BE-NEXT: ushll v0.8h, v0.8b, #0347; CHECK-BE-NEXT: ushll2 v2.4s, v1.8h, #0348; CHECK-BE-NEXT: ushll v1.4s, v1.4h, #0349; CHECK-BE-NEXT: ushll2 v3.4s, v0.8h, #0350; CHECK-BE-NEXT: ushll v0.4s, v0.4h, #0351; CHECK-BE-NEXT: st1 { v2.4s }, [x8]352; CHECK-BE-NEXT: add x8, x1, #32353; CHECK-BE-NEXT: st1 { v1.4s }, [x8]354; CHECK-BE-NEXT: add x8, x1, #16355; CHECK-BE-NEXT: st1 { v3.4s }, [x8]356; CHECK-BE-NEXT: st1 { v0.4s }, [x1]357; CHECK-BE-NEXT: ret358entry:359 %load = load <16 x i8>, ptr %src360 %ext = zext <16 x i8> %load to <16 x i32>361 store <16 x i32> %ext, ptr %dst362 ret void363}364 365; Avoid using tbl when optimizing for size.366define void @zext_v16i8_to_v16i32_in_loop_optsize(ptr %src, ptr %dst) optsize {367; CHECK-LABEL: zext_v16i8_to_v16i32_in_loop_optsize:368; CHECK: ; %bb.0: ; %entry369; CHECK-NEXT: mov x8, xzr370; CHECK-NEXT: LBB3_1: ; %loop371; CHECK-NEXT: ; =>This Inner Loop Header: Depth=1372; CHECK-NEXT: ldr q0, [x0, x8]373; CHECK-NEXT: add x8, x8, #16374; CHECK-NEXT: cmp x8, #128375; CHECK-NEXT: ushll2.8h v1, v0, #0376; CHECK-NEXT: ushll.8h v0, v0, #0377; CHECK-NEXT: ushll2.4s v2, v1, #0378; CHECK-NEXT: ushll.4s v1, v1, #0379; CHECK-NEXT: ushll2.4s v3, v0, #0380; CHECK-NEXT: ushll.4s v0, v0, #0381; CHECK-NEXT: stp q1, q2, [x1, #32]382; CHECK-NEXT: stp q0, q3, [x1], #64383; CHECK-NEXT: b.ne LBB3_1384; CHECK-NEXT: ; %bb.2: ; %exit385; CHECK-NEXT: ret386;387; CHECK-BE-LABEL: zext_v16i8_to_v16i32_in_loop_optsize:388; CHECK-BE: // %bb.0: // %entry389; CHECK-BE-NEXT: mov x8, xzr390; CHECK-BE-NEXT: .LBB3_1: // %loop391; CHECK-BE-NEXT: // =>This Inner Loop Header: Depth=1392; CHECK-BE-NEXT: add x9, x0, x8393; CHECK-BE-NEXT: add x8, x8, #16394; CHECK-BE-NEXT: ld1 { v0.16b }, [x9]395; CHECK-BE-NEXT: add x9, x1, #48396; CHECK-BE-NEXT: cmp x8, #128397; CHECK-BE-NEXT: ushll2 v1.8h, v0.16b, #0398; CHECK-BE-NEXT: ushll v0.8h, v0.8b, #0399; CHECK-BE-NEXT: ushll2 v2.4s, v1.8h, #0400; CHECK-BE-NEXT: ushll v1.4s, v1.4h, #0401; CHECK-BE-NEXT: ushll2 v3.4s, v0.8h, #0402; CHECK-BE-NEXT: ushll v0.4s, v0.4h, #0403; CHECK-BE-NEXT: st1 { v2.4s }, [x9]404; CHECK-BE-NEXT: add x9, x1, #32405; CHECK-BE-NEXT: st1 { v1.4s }, [x9]406; CHECK-BE-NEXT: add x9, x1, #16407; CHECK-BE-NEXT: st1 { v0.4s }, [x1]408; CHECK-BE-NEXT: add x1, x1, #64409; CHECK-BE-NEXT: st1 { v3.4s }, [x9]410; CHECK-BE-NEXT: b.ne .LBB3_1411; CHECK-BE-NEXT: // %bb.2: // %exit412; CHECK-BE-NEXT: ret413entry:414 br label %loop415 416loop:417 %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]418 %src.gep = getelementptr i8, ptr %src, i64 %iv419 %load = load <16 x i8>, ptr %src.gep420 %ext = zext <16 x i8> %load to <16 x i32>421 %dst.gep = getelementptr i32, ptr %dst, i64 %iv422 store <16 x i32> %ext, ptr %dst.gep423 %iv.next = add nuw i64 %iv, 16424 %ec = icmp eq i64 %iv.next, 128425 br i1 %ec, label %exit, label %loop426 427exit:428 ret void429}430 431; Avoid using tbl when optimizing for size.432define void @zext_v16i8_to_v16i32_in_loop_minsize(ptr %src, ptr %dst) minsize {433; CHECK-LABEL: zext_v16i8_to_v16i32_in_loop_minsize:434; CHECK: ; %bb.0: ; %entry435; CHECK-NEXT: mov x8, xzr436; CHECK-NEXT: LBB4_1: ; %loop437; CHECK-NEXT: ; =>This Inner Loop Header: Depth=1438; CHECK-NEXT: ldr q0, [x0, x8]439; CHECK-NEXT: add x8, x8, #16440; CHECK-NEXT: cmp x8, #128441; CHECK-NEXT: ushll2.8h v1, v0, #0442; CHECK-NEXT: ushll.8h v0, v0, #0443; CHECK-NEXT: ushll2.4s v2, v1, #0444; CHECK-NEXT: ushll.4s v1, v1, #0445; CHECK-NEXT: ushll2.4s v3, v0, #0446; CHECK-NEXT: ushll.4s v0, v0, #0447; CHECK-NEXT: stp q1, q2, [x1, #32]448; CHECK-NEXT: stp q0, q3, [x1], #64449; CHECK-NEXT: b.ne LBB4_1450; CHECK-NEXT: ; %bb.2: ; %exit451; CHECK-NEXT: ret452;453; CHECK-BE-LABEL: zext_v16i8_to_v16i32_in_loop_minsize:454; CHECK-BE: // %bb.0: // %entry455; CHECK-BE-NEXT: mov x8, xzr456; CHECK-BE-NEXT: .LBB4_1: // %loop457; CHECK-BE-NEXT: // =>This Inner Loop Header: Depth=1458; CHECK-BE-NEXT: add x9, x0, x8459; CHECK-BE-NEXT: add x8, x8, #16460; CHECK-BE-NEXT: ld1 { v0.16b }, [x9]461; CHECK-BE-NEXT: add x9, x1, #48462; CHECK-BE-NEXT: cmp x8, #128463; CHECK-BE-NEXT: ushll2 v1.8h, v0.16b, #0464; CHECK-BE-NEXT: ushll v0.8h, v0.8b, #0465; CHECK-BE-NEXT: ushll2 v2.4s, v1.8h, #0466; CHECK-BE-NEXT: ushll v1.4s, v1.4h, #0467; CHECK-BE-NEXT: ushll2 v3.4s, v0.8h, #0468; CHECK-BE-NEXT: ushll v0.4s, v0.4h, #0469; CHECK-BE-NEXT: st1 { v2.4s }, [x9]470; CHECK-BE-NEXT: add x9, x1, #32471; CHECK-BE-NEXT: st1 { v1.4s }, [x9]472; CHECK-BE-NEXT: add x9, x1, #16473; CHECK-BE-NEXT: st1 { v0.4s }, [x1]474; CHECK-BE-NEXT: add x1, x1, #64475; CHECK-BE-NEXT: st1 { v3.4s }, [x9]476; CHECK-BE-NEXT: b.ne .LBB4_1477; CHECK-BE-NEXT: // %bb.2: // %exit478; CHECK-BE-NEXT: ret479entry:480 br label %loop481 482loop:483 %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]484 %src.gep = getelementptr i8, ptr %src, i64 %iv485 %load = load <16 x i8>, ptr %src.gep486 %ext = zext <16 x i8> %load to <16 x i32>487 %dst.gep = getelementptr i32, ptr %dst, i64 %iv488 store <16 x i32> %ext, ptr %dst.gep489 %iv.next = add nuw i64 %iv, 16490 %ec = icmp eq i64 %iv.next, 128491 br i1 %ec, label %exit, label %loop492 493exit:494 ret void495}496 497define void @zext_v16i8_to_v16i16_in_loop(ptr %src, ptr %dst) {498; CHECK-LABEL: zext_v16i8_to_v16i16_in_loop:499; CHECK: ; %bb.0: ; %entry500; CHECK-NEXT: mov x8, xzr501; CHECK-NEXT: LBB5_1: ; %loop502; CHECK-NEXT: ; =>This Inner Loop Header: Depth=1503; CHECK-NEXT: ldr q0, [x0, x8]504; CHECK-NEXT: add x8, x8, #16505; CHECK-NEXT: cmp x8, #128506; CHECK-NEXT: ushll2.8h v1, v0, #0507; CHECK-NEXT: ushll.8h v0, v0, #0508; CHECK-NEXT: stp q0, q1, [x1], #32509; CHECK-NEXT: b.ne LBB5_1510; CHECK-NEXT: ; %bb.2: ; %exit511; CHECK-NEXT: ret512;513; CHECK-BE-LABEL: zext_v16i8_to_v16i16_in_loop:514; CHECK-BE: // %bb.0: // %entry515; CHECK-BE-NEXT: mov x8, xzr516; CHECK-BE-NEXT: .LBB5_1: // %loop517; CHECK-BE-NEXT: // =>This Inner Loop Header: Depth=1518; CHECK-BE-NEXT: add x9, x0, x8519; CHECK-BE-NEXT: add x8, x8, #16520; CHECK-BE-NEXT: ld1 { v0.16b }, [x9]521; CHECK-BE-NEXT: add x9, x1, #16522; CHECK-BE-NEXT: cmp x8, #128523; CHECK-BE-NEXT: ushll2 v1.8h, v0.16b, #0524; CHECK-BE-NEXT: ushll v0.8h, v0.8b, #0525; CHECK-BE-NEXT: st1 { v0.8h }, [x1]526; CHECK-BE-NEXT: add x1, x1, #32527; CHECK-BE-NEXT: st1 { v1.8h }, [x9]528; CHECK-BE-NEXT: b.ne .LBB5_1529; CHECK-BE-NEXT: // %bb.2: // %exit530; CHECK-BE-NEXT: ret531 532 533entry:534 br label %loop535 536loop:537 %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]538 %src.gep = getelementptr i8, ptr %src, i64 %iv539 %load = load <16 x i8>, ptr %src.gep540 %ext = zext <16 x i8> %load to <16 x i16>541 %dst.gep = getelementptr i16, ptr %dst, i64 %iv542 store <16 x i16> %ext, ptr %dst.gep543 %iv.next = add nuw i64 %iv, 16544 %ec = icmp eq i64 %iv.next, 128545 br i1 %ec, label %exit, label %loop546 547exit:548 ret void549}550 551; CHECK-LABEL: lCPI6_0:552; CHECK-NEXT: .byte 0 ; 0x0553; CHECK-NEXT: .byte 255 ; 0xff554; CHECK-NEXT: .byte 255 ; 0xff555; CHECK-NEXT: .byte 255 ; 0xff556; CHECK-NEXT: .byte 1 ; 0x1557; CHECK-NEXT: .byte 255 ; 0xff558; CHECK-NEXT: .byte 255 ; 0xff559; CHECK-NEXT: .byte 255 ; 0xff560; CHECK-NEXT: .byte 2 ; 0x2561; CHECK-NEXT: .byte 255 ; 0xff562; CHECK-NEXT: .byte 255 ; 0xff563; CHECK-NEXT: .byte 255 ; 0xff564; CHECK-NEXT: .byte 3 ; 0x3565; CHECK-NEXT: .byte 255 ; 0xff566; CHECK-NEXT: .byte 255 ; 0xff567; CHECK-NEXT: .byte 255 ; 0xff568; CHECK-NEXT: lCPI6_1:569; CHECK-NEXT: .byte 4 ; 0x4570; CHECK-NEXT: .byte 255 ; 0xff571; CHECK-NEXT: .byte 255 ; 0xff572; CHECK-NEXT: .byte 255 ; 0xff573; CHECK-NEXT: .byte 5 ; 0x5574; CHECK-NEXT: .byte 255 ; 0xff575; CHECK-NEXT: .byte 255 ; 0xff576; CHECK-NEXT: .byte 255 ; 0xff577; CHECK-NEXT: .byte 6 ; 0x6578; CHECK-NEXT: .byte 255 ; 0xff579; CHECK-NEXT: .byte 255 ; 0xff580; CHECK-NEXT: .byte 255 ; 0xff581; CHECK-NEXT: .byte 7 ; 0x7582; CHECK-NEXT: .byte 255 ; 0xff583; CHECK-NEXT: .byte 255 ; 0xff584; CHECK-NEXT: .byte 255 ; 0xff585 586; CHECK-BE: .LCPI6_0:587; CHECK-BE-NEXT: .byte 255 // 0xff588; CHECK-BE-NEXT: .byte 255 // 0xff589; CHECK-BE-NEXT: .byte 255 // 0xff590; CHECK-BE-NEXT: .byte 0 // 0x0591; CHECK-BE-NEXT: .byte 255 // 0xff592; CHECK-BE-NEXT: .byte 255 // 0xff593; CHECK-BE-NEXT: .byte 255 // 0xff594; CHECK-BE-NEXT: .byte 1 // 0x1595; CHECK-BE-NEXT: .byte 255 // 0xff596; CHECK-BE-NEXT: .byte 255 // 0xff597; CHECK-BE-NEXT: .byte 255 // 0xff598; CHECK-BE-NEXT: .byte 2 // 0x2599; CHECK-BE-NEXT: .byte 255 // 0xff600; CHECK-BE-NEXT: .byte 255 // 0xff601; CHECK-BE-NEXT: .byte 255 // 0xff602; CHECK-BE-NEXT: .byte 3 // 0x3603; CHECK-BE-NEXT: .LCPI6_1:604; CHECK-BE-NEXT: .byte 255 // 0xff605; CHECK-BE-NEXT: .byte 255 // 0xff606; CHECK-BE-NEXT: .byte 255 // 0xff607; CHECK-BE-NEXT: .byte 4 // 0x4608; CHECK-BE-NEXT: .byte 255 // 0xff609; CHECK-BE-NEXT: .byte 255 // 0xff610; CHECK-BE-NEXT: .byte 255 // 0xff611; CHECK-BE-NEXT: .byte 5 // 0x5612; CHECK-BE-NEXT: .byte 255 // 0xff613; CHECK-BE-NEXT: .byte 255 // 0xff614; CHECK-BE-NEXT: .byte 255 // 0xff615; CHECK-BE-NEXT: .byte 6 // 0x6616; CHECK-BE-NEXT: .byte 255 // 0xff617; CHECK-BE-NEXT: .byte 255 // 0xff618; CHECK-BE-NEXT: .byte 255 // 0xff619; CHECK-BE-NEXT: .byte 7 // 0x7620 621define void @zext_v8i8_to_v8i32_in_loop(ptr %src, ptr %dst) {622; CHECK-LABEL: zext_v8i8_to_v8i32_in_loop:623; CHECK: ; %bb.0: ; %entry624; CHECK-NEXT: Lloh8:625; CHECK-NEXT: adrp x8, lCPI6_0@PAGE626; CHECK-NEXT: Lloh9:627; CHECK-NEXT: adrp x9, lCPI6_1@PAGE628; CHECK-NEXT: Lloh10:629; CHECK-NEXT: ldr q0, [x8, lCPI6_0@PAGEOFF]630; CHECK-NEXT: Lloh11:631; CHECK-NEXT: ldr q1, [x9, lCPI6_1@PAGEOFF]632; CHECK-NEXT: mov x8, xzr633; CHECK-NEXT: LBB6_1: ; %loop634; CHECK-NEXT: ; =>This Inner Loop Header: Depth=1635; CHECK-NEXT: ldr d2, [x0, x8]636; CHECK-NEXT: add x8, x8, #16637; CHECK-NEXT: cmp x8, #128638; CHECK-NEXT: tbl.16b v3, { v2 }, v1639; CHECK-NEXT: tbl.16b v2, { v2 }, v0640; CHECK-NEXT: stp q2, q3, [x1], #64641; CHECK-NEXT: b.ne LBB6_1642; CHECK-NEXT: ; %bb.2: ; %exit643; CHECK-NEXT: ret644; CHECK-NEXT: .loh AdrpLdr Lloh9, Lloh11645; CHECK-NEXT: .loh AdrpLdr Lloh8, Lloh10646;647; CHECK-BE-LABEL: zext_v8i8_to_v8i32_in_loop:648; CHECK-BE: // %bb.0: // %entry649; CHECK-BE-NEXT: adrp x8, .LCPI6_0650; CHECK-BE-NEXT: add x8, x8, :lo12:.LCPI6_0651; CHECK-BE-NEXT: ld1 { v0.16b }, [x8]652; CHECK-BE-NEXT: adrp x8, .LCPI6_1653; CHECK-BE-NEXT: add x8, x8, :lo12:.LCPI6_1654; CHECK-BE-NEXT: ld1 { v1.16b }, [x8]655; CHECK-BE-NEXT: mov x8, xzr656; CHECK-BE-NEXT: .LBB6_1: // %loop657; CHECK-BE-NEXT: // =>This Inner Loop Header: Depth=1658; CHECK-BE-NEXT: add x9, x0, x8659; CHECK-BE-NEXT: add x8, x8, #16660; CHECK-BE-NEXT: ld1 { v2.8b }, [x9]661; CHECK-BE-NEXT: add x9, x1, #16662; CHECK-BE-NEXT: cmp x8, #128663; CHECK-BE-NEXT: tbl v3.16b, { v2.16b }, v1.16b664; CHECK-BE-NEXT: tbl v2.16b, { v2.16b }, v0.16b665; CHECK-BE-NEXT: st1 { v2.16b }, [x1]666; CHECK-BE-NEXT: add x1, x1, #64667; CHECK-BE-NEXT: st1 { v3.16b }, [x9]668; CHECK-BE-NEXT: b.ne .LBB6_1669; CHECK-BE-NEXT: // %bb.2: // %exit670; CHECK-BE-NEXT: ret671entry:672 br label %loop673 674loop:675 %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]676 %src.gep = getelementptr i8, ptr %src, i64 %iv677 %load = load <8 x i8>, ptr %src.gep678 %ext = zext <8 x i8> %load to <8 x i32>679 %dst.gep = getelementptr i32, ptr %dst, i64 %iv680 store <8 x i32> %ext, ptr %dst.gep681 %iv.next = add nuw i64 %iv, 16682 %ec = icmp eq i64 %iv.next, 128683 br i1 %ec, label %exit, label %loop684 685exit:686 ret void687}688 689define void @zext_v16i8_to_v16i64_in_loop(ptr %src, ptr %dst) {690; CHECK-LABEL: zext_v16i8_to_v16i64_in_loop:691; CHECK: ; %bb.0: ; %entry692; CHECK-NEXT: mov x8, xzr693; CHECK-NEXT: LBB7_1: ; %loop694; CHECK-NEXT: ; =>This Inner Loop Header: Depth=1695; CHECK-NEXT: ldr q0, [x0, x8]696; CHECK-NEXT: add x8, x8, #16697; CHECK-NEXT: cmp x8, #128698; CHECK-NEXT: ushll2.8h v1, v0, #0699; CHECK-NEXT: ushll.8h v0, v0, #0700; CHECK-NEXT: ushll2.4s v2, v1, #0701; CHECK-NEXT: ushll.4s v1, v1, #0702; CHECK-NEXT: ushll2.4s v4, v0, #0703; CHECK-NEXT: ushll.4s v0, v0, #0704; CHECK-NEXT: ushll2.2d v3, v2, #0705; CHECK-NEXT: ushll.2d v2, v2, #0706; CHECK-NEXT: ushll2.2d v5, v1, #0707; CHECK-NEXT: ushll.2d v1, v1, #0708; CHECK-NEXT: stp q2, q3, [x1, #96]709; CHECK-NEXT: ushll2.2d v3, v4, #0710; CHECK-NEXT: ushll.2d v2, v4, #0711; CHECK-NEXT: ushll2.2d v4, v0, #0712; CHECK-NEXT: ushll.2d v0, v0, #0713; CHECK-NEXT: stp q1, q5, [x1, #64]714; CHECK-NEXT: stp q2, q3, [x1, #32]715; CHECK-NEXT: stp q0, q4, [x1], #128716; CHECK-NEXT: b.ne LBB7_1717; CHECK-NEXT: ; %bb.2: ; %exit718; CHECK-NEXT: ret719;720; CHECK-BE-LABEL: zext_v16i8_to_v16i64_in_loop:721; CHECK-BE: // %bb.0: // %entry722; CHECK-BE-NEXT: mov x8, xzr723; CHECK-BE-NEXT: .LBB7_1: // %loop724; CHECK-BE-NEXT: // =>This Inner Loop Header: Depth=1725; CHECK-BE-NEXT: add x9, x0, x8726; CHECK-BE-NEXT: add x8, x8, #16727; CHECK-BE-NEXT: ld1 { v0.16b }, [x9]728; CHECK-BE-NEXT: add x9, x1, #112729; CHECK-BE-NEXT: cmp x8, #128730; CHECK-BE-NEXT: ushll2 v1.8h, v0.16b, #0731; CHECK-BE-NEXT: ushll v0.8h, v0.8b, #0732; CHECK-BE-NEXT: ushll2 v2.4s, v1.8h, #0733; CHECK-BE-NEXT: ushll v1.4s, v1.4h, #0734; CHECK-BE-NEXT: ushll2 v4.4s, v0.8h, #0735; CHECK-BE-NEXT: ushll v0.4s, v0.4h, #0736; CHECK-BE-NEXT: ushll2 v3.2d, v2.4s, #0737; CHECK-BE-NEXT: ushll v2.2d, v2.2s, #0738; CHECK-BE-NEXT: ushll2 v5.2d, v1.4s, #0739; CHECK-BE-NEXT: ushll v1.2d, v1.2s, #0740; CHECK-BE-NEXT: st1 { v3.2d }, [x9]741; CHECK-BE-NEXT: add x9, x1, #96742; CHECK-BE-NEXT: ushll2 v3.2d, v4.4s, #0743; CHECK-BE-NEXT: st1 { v2.2d }, [x9]744; CHECK-BE-NEXT: add x9, x1, #80745; CHECK-BE-NEXT: ushll v2.2d, v4.2s, #0746; CHECK-BE-NEXT: st1 { v5.2d }, [x9]747; CHECK-BE-NEXT: add x9, x1, #48748; CHECK-BE-NEXT: st1 { v3.2d }, [x9]749; CHECK-BE-NEXT: ushll2 v3.2d, v0.4s, #0750; CHECK-BE-NEXT: ushll v0.2d, v0.2s, #0751; CHECK-BE-NEXT: add x9, x1, #64752; CHECK-BE-NEXT: st1 { v1.2d }, [x9]753; CHECK-BE-NEXT: add x9, x1, #32754; CHECK-BE-NEXT: st1 { v2.2d }, [x9]755; CHECK-BE-NEXT: add x9, x1, #16756; CHECK-BE-NEXT: st1 { v0.2d }, [x1]757; CHECK-BE-NEXT: add x1, x1, #128758; CHECK-BE-NEXT: st1 { v3.2d }, [x9]759; CHECK-BE-NEXT: b.ne .LBB7_1760; CHECK-BE-NEXT: // %bb.2: // %exit761; CHECK-BE-NEXT: ret762 763 764entry:765 br label %loop766 767loop:768 %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]769 %src.gep = getelementptr i8, ptr %src, i64 %iv770 %load = load <16 x i8>, ptr %src.gep771 %ext = zext <16 x i8> %load to <16 x i64>772 %dst.gep = getelementptr i64, ptr %dst, i64 %iv773 store <16 x i64> %ext, ptr %dst.gep774 %iv.next = add nuw i64 %iv, 16775 %ec = icmp eq i64 %iv.next, 128776 br i1 %ec, label %exit, label %loop777 778exit:779 ret void780}781 782define void @zext_v8i8_to_v8i64_in_loop(ptr %src, ptr %dst) {783; CHECK-LABEL: zext_v8i8_to_v8i64_in_loop:784; CHECK: ; %bb.0: ; %entry785; CHECK-NEXT: mov x8, xzr786; CHECK-NEXT: LBB8_1: ; %loop787; CHECK-NEXT: ; =>This Inner Loop Header: Depth=1788; CHECK-NEXT: ldr d0, [x0, x8]789; CHECK-NEXT: add x8, x8, #16790; CHECK-NEXT: cmp x8, #128791; CHECK-NEXT: ushll.8h v0, v0, #0792; CHECK-NEXT: ushll2.4s v1, v0, #0793; CHECK-NEXT: ushll.4s v0, v0, #0794; CHECK-NEXT: ushll2.2d v2, v1, #0795; CHECK-NEXT: ushll.2d v1, v1, #0796; CHECK-NEXT: ushll2.2d v3, v0, #0797; CHECK-NEXT: ushll.2d v0, v0, #0798; CHECK-NEXT: stp q1, q2, [x1, #32]799; CHECK-NEXT: stp q0, q3, [x1], #128800; CHECK-NEXT: b.ne LBB8_1801; CHECK-NEXT: ; %bb.2: ; %exit802; CHECK-NEXT: ret803;804; CHECK-BE-LABEL: zext_v8i8_to_v8i64_in_loop:805; CHECK-BE: // %bb.0: // %entry806; CHECK-BE-NEXT: mov x8, xzr807; CHECK-BE-NEXT: .LBB8_1: // %loop808; CHECK-BE-NEXT: // =>This Inner Loop Header: Depth=1809; CHECK-BE-NEXT: add x9, x0, x8810; CHECK-BE-NEXT: add x8, x8, #16811; CHECK-BE-NEXT: ld1 { v0.8b }, [x9]812; CHECK-BE-NEXT: add x9, x1, #48813; CHECK-BE-NEXT: cmp x8, #128814; CHECK-BE-NEXT: ushll v0.8h, v0.8b, #0815; CHECK-BE-NEXT: ushll2 v1.4s, v0.8h, #0816; CHECK-BE-NEXT: ushll v0.4s, v0.4h, #0817; CHECK-BE-NEXT: ushll2 v2.2d, v1.4s, #0818; CHECK-BE-NEXT: ushll v1.2d, v1.2s, #0819; CHECK-BE-NEXT: ushll2 v3.2d, v0.4s, #0820; CHECK-BE-NEXT: ushll v0.2d, v0.2s, #0821; CHECK-BE-NEXT: st1 { v2.2d }, [x9]822; CHECK-BE-NEXT: add x9, x1, #32823; CHECK-BE-NEXT: st1 { v1.2d }, [x9]824; CHECK-BE-NEXT: add x9, x1, #16825; CHECK-BE-NEXT: st1 { v0.2d }, [x1]826; CHECK-BE-NEXT: add x1, x1, #128827; CHECK-BE-NEXT: st1 { v3.2d }, [x9]828; CHECK-BE-NEXT: b.ne .LBB8_1829; CHECK-BE-NEXT: // %bb.2: // %exit830; CHECK-BE-NEXT: ret831 832 833entry:834 br label %loop835 836loop:837 %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]838 %src.gep = getelementptr i8, ptr %src, i64 %iv839 %load = load <8 x i8>, ptr %src.gep840 %ext = zext <8 x i8> %load to <8 x i64>841 %dst.gep = getelementptr i64, ptr %dst, i64 %iv842 store <8 x i64> %ext, ptr %dst.gep843 %iv.next = add nuw i64 %iv, 16844 %ec = icmp eq i64 %iv.next, 128845 br i1 %ec, label %exit, label %loop846 847exit:848 ret void849}850 851define void @zext_v8i8_to_v8i16_in_loop(ptr %src, ptr %dst) {852; CHECK-LABEL: zext_v8i8_to_v8i16_in_loop:853; CHECK: ; %bb.0: ; %entry854; CHECK-NEXT: mov x8, xzr855; CHECK-NEXT: LBB9_1: ; %loop856; CHECK-NEXT: ; =>This Inner Loop Header: Depth=1857; CHECK-NEXT: ldr d0, [x0, x8]858; CHECK-NEXT: add x8, x8, #16859; CHECK-NEXT: cmp x8, #128860; CHECK-NEXT: ushll.8h v0, v0, #0861; CHECK-NEXT: str q0, [x1], #32862; CHECK-NEXT: b.ne LBB9_1863; CHECK-NEXT: ; %bb.2: ; %exit864; CHECK-NEXT: ret865;866; CHECK-BE-LABEL: zext_v8i8_to_v8i16_in_loop:867; CHECK-BE: // %bb.0: // %entry868; CHECK-BE-NEXT: mov x8, xzr869; CHECK-BE-NEXT: .LBB9_1: // %loop870; CHECK-BE-NEXT: // =>This Inner Loop Header: Depth=1871; CHECK-BE-NEXT: add x9, x0, x8872; CHECK-BE-NEXT: add x8, x8, #16873; CHECK-BE-NEXT: ld1 { v0.8b }, [x9]874; CHECK-BE-NEXT: cmp x8, #128875; CHECK-BE-NEXT: ushll v0.8h, v0.8b, #0876; CHECK-BE-NEXT: st1 { v0.8h }, [x1]877; CHECK-BE-NEXT: add x1, x1, #32878; CHECK-BE-NEXT: b.ne .LBB9_1879; CHECK-BE-NEXT: // %bb.2: // %exit880; CHECK-BE-NEXT: ret881 882 883 884entry:885 br label %loop886 887loop:888 %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]889 %src.gep = getelementptr i8, ptr %src, i64 %iv890 %load = load <8 x i8>, ptr %src.gep891 %ext = zext <8 x i8> %load to <8 x i16>892 %dst.gep = getelementptr i16, ptr %dst, i64 %iv893 store <8 x i16> %ext, ptr %dst.gep894 %iv.next = add nuw i64 %iv, 16895 %ec = icmp eq i64 %iv.next, 128896 br i1 %ec, label %exit, label %loop897 898exit:899 ret void900}901 902define void @zext_v8i8_to_v8i20_in_loop(ptr %src, ptr %dst) {903; CHECK-LABEL: zext_v8i8_to_v8i20_in_loop:904; CHECK: ; %bb.0: ; %entry905; CHECK-NEXT: mov x8, xzr906; CHECK-NEXT: LBB10_1: ; %loop907; CHECK-NEXT: ; =>This Inner Loop Header: Depth=1908; CHECK-NEXT: ldr d0, [x0, x8]909; CHECK-NEXT: add x8, x8, #16910; CHECK-NEXT: cmp x8, #128911; CHECK-NEXT: ushll.8h v0, v0, #0912; CHECK-NEXT: ushll2.4s v1, v0, #0913; CHECK-NEXT: ushll.4s v0, v0, #0914; CHECK-NEXT: mov.s w9, v1[1]915; CHECK-NEXT: mov.s w11, v0[1]916; CHECK-NEXT: fmov w10, s1917; CHECK-NEXT: fmov w14, s0918; CHECK-NEXT: mov.s w13, v1[2]919; CHECK-NEXT: mov.s w16, v0[2]920; CHECK-NEXT: mov.s w12, v1[3]921; CHECK-NEXT: mov.s w15, v0[3]922; CHECK-NEXT: orr x9, x10, x9, lsl #20923; CHECK-NEXT: orr x10, x14, x11, lsl #20924; CHECK-NEXT: orr x9, x9, x13, lsl #40925; CHECK-NEXT: orr x10, x10, x16, lsl #40926; CHECK-NEXT: lsr w11, w12, #4927; CHECK-NEXT: lsr w13, w15, #4928; CHECK-NEXT: orr x9, x9, x12, lsl #60929; CHECK-NEXT: orr x10, x10, x15, lsl #60930; CHECK-NEXT: strh w11, [x1, #18]931; CHECK-NEXT: strh w13, [x1, #8]932; CHECK-NEXT: stur x9, [x1, #10]933; CHECK-NEXT: str x10, [x1], #64934; CHECK-NEXT: b.ne LBB10_1935; CHECK-NEXT: ; %bb.2: ; %exit936; CHECK-NEXT: ret937;938; CHECK-BE-LABEL: zext_v8i8_to_v8i20_in_loop:939; CHECK-BE: // %bb.0: // %entry940; CHECK-BE-NEXT: mov x8, xzr941; CHECK-BE-NEXT: .LBB10_1: // %loop942; CHECK-BE-NEXT: // =>This Inner Loop Header: Depth=1943; CHECK-BE-NEXT: add x9, x0, x8944; CHECK-BE-NEXT: add x8, x8, #16945; CHECK-BE-NEXT: ld1 { v0.8b }, [x9]946; CHECK-BE-NEXT: cmp x8, #128947; CHECK-BE-NEXT: ushll v0.8h, v0.8b, #0948; CHECK-BE-NEXT: ushll2 v1.4s, v0.8h, #0949; CHECK-BE-NEXT: ushll v0.4s, v0.4h, #0950; CHECK-BE-NEXT: mov w9, v1.s[1]951; CHECK-BE-NEXT: mov w10, v0.s[1]952; CHECK-BE-NEXT: fmov w12, s1953; CHECK-BE-NEXT: fmov w14, s0954; CHECK-BE-NEXT: mov w11, v1.s[2]955; CHECK-BE-NEXT: mov w13, v0.s[2]956; CHECK-BE-NEXT: mov w15, v1.s[3]957; CHECK-BE-NEXT: lsl x9, x9, #40958; CHECK-BE-NEXT: lsl x10, x10, #40959; CHECK-BE-NEXT: orr x9, x9, x12, lsl #60960; CHECK-BE-NEXT: orr x10, x10, x14, lsl #60961; CHECK-BE-NEXT: lsr x12, x12, #4962; CHECK-BE-NEXT: strh w15, [x1, #18]963; CHECK-BE-NEXT: orr x9, x9, x11, lsl #20964; CHECK-BE-NEXT: orr x10, x10, x13, lsl #20965; CHECK-BE-NEXT: mov w11, v0.s[3]966; CHECK-BE-NEXT: lsr x13, x14, #4967; CHECK-BE-NEXT: lsr x9, x9, #16968; CHECK-BE-NEXT: lsr x10, x10, #16969; CHECK-BE-NEXT: bfi x9, x12, #48, #4970; CHECK-BE-NEXT: bfi x10, x13, #48, #4971; CHECK-BE-NEXT: strh w11, [x1, #8]972; CHECK-BE-NEXT: stur x9, [x1, #10]973; CHECK-BE-NEXT: str x10, [x1], #64974; CHECK-BE-NEXT: b.ne .LBB10_1975; CHECK-BE-NEXT: // %bb.2: // %exit976; CHECK-BE-NEXT: ret977 978 979entry:980 br label %loop981 982loop:983 %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]984 %src.gep = getelementptr i8, ptr %src, i64 %iv985 %load = load <8 x i8>, ptr %src.gep986 %ext = zext <8 x i8> %load to <8 x i20>987 %dst.gep = getelementptr i20, ptr %dst, i64 %iv988 store <8 x i20> %ext, ptr %dst.gep989 %iv.next = add nuw i64 %iv, 16990 %ec = icmp eq i64 %iv.next, 128991 br i1 %ec, label %exit, label %loop992 993exit:994 ret void995}996 997define void @zext_v4i8_to_v4i32_in_loop(ptr %src, ptr %dst) {998; CHECK-LABEL: zext_v4i8_to_v4i32_in_loop:999; CHECK: ; %bb.0: ; %entry1000; CHECK-NEXT: mov x8, xzr1001; CHECK-NEXT: LBB11_1: ; %loop1002; CHECK-NEXT: ; =>This Inner Loop Header: Depth=11003; CHECK-NEXT: ldr s0, [x0, x8]1004; CHECK-NEXT: add x8, x8, #161005; CHECK-NEXT: cmp x8, #1281006; CHECK-NEXT: ushll.8h v0, v0, #01007; CHECK-NEXT: ushll.4s v0, v0, #01008; CHECK-NEXT: str q0, [x1], #641009; CHECK-NEXT: b.ne LBB11_11010; CHECK-NEXT: ; %bb.2: ; %exit1011; CHECK-NEXT: ret1012;1013; CHECK-BE-LABEL: zext_v4i8_to_v4i32_in_loop:1014; CHECK-BE: // %bb.0: // %entry1015; CHECK-BE-NEXT: adrp x8, .LCPI11_01016; CHECK-BE-NEXT: add x8, x8, :lo12:.LCPI11_01017; CHECK-BE-NEXT: ld1 { v0.16b }, [x8]1018; CHECK-BE-NEXT: mov x8, xzr1019; CHECK-BE-NEXT: .LBB11_1: // %loop1020; CHECK-BE-NEXT: // =>This Inner Loop Header: Depth=11021; CHECK-BE-NEXT: ldr s1, [x0, x8]1022; CHECK-BE-NEXT: add x8, x8, #161023; CHECK-BE-NEXT: cmp x8, #1281024; CHECK-BE-NEXT: rev32 v1.16b, v1.16b1025; CHECK-BE-NEXT: tbl v1.16b, { v1.16b }, v0.16b1026; CHECK-BE-NEXT: st1 { v1.16b }, [x1]1027; CHECK-BE-NEXT: add x1, x1, #641028; CHECK-BE-NEXT: b.ne .LBB11_11029; CHECK-BE-NEXT: // %bb.2: // %exit1030; CHECK-BE-NEXT: ret1031 1032 1033entry:1034 br label %loop1035 1036loop:1037 %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]1038 %src.gep = getelementptr i8, ptr %src, i64 %iv1039 %load = load <4 x i8>, ptr %src.gep1040 %ext = zext <4 x i8> %load to <4 x i32>1041 %dst.gep = getelementptr i32, ptr %dst, i64 %iv1042 store <4 x i32> %ext, ptr %dst.gep1043 %iv.next = add nuw i64 %iv, 161044 %ec = icmp eq i64 %iv.next, 1281045 br i1 %ec, label %exit, label %loop1046 1047exit:1048 ret void1049}1050 1051; CHECK-LABEL: lCPI12_0:1052; CHECK-NEXT: .byte 0 ; 0x01053; CHECK-NEXT: .byte 255 ; 0xff1054; CHECK-NEXT: .byte 255 ; 0xff1055; CHECK-NEXT: .byte 255 ; 0xff1056; CHECK-NEXT: .byte 1 ; 0x11057; CHECK-NEXT: .byte 255 ; 0xff1058; CHECK-NEXT: .byte 255 ; 0xff1059; CHECK-NEXT: .byte 255 ; 0xff1060; CHECK-NEXT: .byte 2 ; 0x21061; CHECK-NEXT: .byte 255 ; 0xff1062; CHECK-NEXT: .byte 255 ; 0xff1063; CHECK-NEXT: .byte 255 ; 0xff1064; CHECK-NEXT: .byte 3 ; 0x31065; CHECK-NEXT: .byte 255 ; 0xff1066; CHECK-NEXT: .byte 255 ; 0xff1067; CHECK-NEXT: .byte 255 ; 0xff1068; CHECK-NEXT: lCPI12_1:1069; CHECK-NEXT: .byte 4 ; 0x41070; CHECK-NEXT: .byte 255 ; 0xff1071; CHECK-NEXT: .byte 255 ; 0xff1072; CHECK-NEXT: .byte 255 ; 0xff1073; CHECK-NEXT: .byte 5 ; 0x51074; CHECK-NEXT: .byte 255 ; 0xff1075; CHECK-NEXT: .byte 255 ; 0xff1076; CHECK-NEXT: .byte 255 ; 0xff1077; CHECK-NEXT: .byte 6 ; 0x61078; CHECK-NEXT: .byte 255 ; 0xff1079; CHECK-NEXT: .byte 255 ; 0xff1080; CHECK-NEXT: .byte 255 ; 0xff1081; CHECK-NEXT: .byte 7 ; 0x71082; CHECK-NEXT: .byte 255 ; 0xff1083; CHECK-NEXT: .byte 255 ; 0xff1084; CHECK-NEXT: .byte 255 ; 0xff1085; CHECK-NEXT: lCPI12_2:1086; CHECK-NEXT: .byte 8 ; 0x81087; CHECK-NEXT: .byte 255 ; 0xff1088; CHECK-NEXT: .byte 255 ; 0xff1089; CHECK-NEXT: .byte 255 ; 0xff1090; CHECK-NEXT: .byte 9 ; 0x91091; CHECK-NEXT: .byte 255 ; 0xff1092; CHECK-NEXT: .byte 255 ; 0xff1093; CHECK-NEXT: .byte 255 ; 0xff1094; CHECK-NEXT: .byte 10 ; 0xa1095; CHECK-NEXT: .byte 255 ; 0xff1096; CHECK-NEXT: .byte 255 ; 0xff1097; CHECK-NEXT: .byte 255 ; 0xff1098; CHECK-NEXT: .byte 11 ; 0xb1099; CHECK-NEXT: .byte 255 ; 0xff1100; CHECK-NEXT: .byte 255 ; 0xff1101; CHECK-NEXT: .byte 255 ; 0xff1102 1103; CHECK-BE-LABEL: .LCPI12_0:1104; CHECK-BE-NEXT: .byte 255 // 0xff1105; CHECK-BE-NEXT: .byte 255 // 0xff1106; CHECK-BE-NEXT: .byte 255 // 0xff1107; CHECK-BE-NEXT: .byte 0 // 0x01108; CHECK-BE-NEXT: .byte 255 // 0xff1109; CHECK-BE-NEXT: .byte 255 // 0xff1110; CHECK-BE-NEXT: .byte 255 // 0xff1111; CHECK-BE-NEXT: .byte 1 // 0x11112; CHECK-BE-NEXT: .byte 255 // 0xff1113; CHECK-BE-NEXT: .byte 255 // 0xff1114; CHECK-BE-NEXT: .byte 255 // 0xff1115; CHECK-BE-NEXT: .byte 2 // 0x21116; CHECK-BE-NEXT: .byte 255 // 0xff1117; CHECK-BE-NEXT: .byte 255 // 0xff1118; CHECK-BE-NEXT: .byte 255 // 0xff1119; CHECK-BE-NEXT: .byte 3 // 0x31120; CHECK-BE-NEXT: .LCPI12_1:1121; CHECK-BE-NEXT: .byte 255 // 0xff1122; CHECK-BE-NEXT: .byte 255 // 0xff1123; CHECK-BE-NEXT: .byte 255 // 0xff1124; CHECK-BE-NEXT: .byte 4 // 0x41125; CHECK-BE-NEXT: .byte 255 // 0xff1126; CHECK-BE-NEXT: .byte 255 // 0xff1127; CHECK-BE-NEXT: .byte 255 // 0xff1128; CHECK-BE-NEXT: .byte 5 // 0x51129; CHECK-BE-NEXT: .byte 255 // 0xff1130; CHECK-BE-NEXT: .byte 255 // 0xff1131; CHECK-BE-NEXT: .byte 255 // 0xff1132; CHECK-BE-NEXT: .byte 6 // 0x61133; CHECK-BE-NEXT: .byte 255 // 0xff1134; CHECK-BE-NEXT: .byte 255 // 0xff1135; CHECK-BE-NEXT: .byte 255 // 0xff1136; CHECK-BE-NEXT: .byte 7 // 0x71137; CHECK-BE-NEXT: .LCPI12_2:1138; CHECK-BE-NEXT: .byte 255 // 0xff1139; CHECK-BE-NEXT: .byte 255 // 0xff1140; CHECK-BE-NEXT: .byte 255 // 0xff1141; CHECK-BE-NEXT: .byte 8 // 0x81142; CHECK-BE-NEXT: .byte 255 // 0xff1143; CHECK-BE-NEXT: .byte 255 // 0xff1144; CHECK-BE-NEXT: .byte 255 // 0xff1145; CHECK-BE-NEXT: .byte 9 // 0x91146; CHECK-BE-NEXT: .byte 255 // 0xff1147; CHECK-BE-NEXT: .byte 255 // 0xff1148; CHECK-BE-NEXT: .byte 255 // 0xff1149; CHECK-BE-NEXT: .byte 10 // 0xa1150; CHECK-BE-NEXT: .byte 255 // 0xff1151; CHECK-BE-NEXT: .byte 255 // 0xff1152; CHECK-BE-NEXT: .byte 255 // 0xff1153; CHECK-BE-NEXT: .byte 11 // 0xb1154 1155define void @zext_v12i8_to_v12i32_in_loop(ptr %src, ptr %dst) {1156; CHECK-LABEL: zext_v12i8_to_v12i32_in_loop:1157; CHECK: ; %bb.0: ; %entry1158; CHECK-NEXT: Lloh12:1159; CHECK-NEXT: adrp x8, lCPI12_0@PAGE1160; CHECK-NEXT: Lloh13:1161; CHECK-NEXT: adrp x9, lCPI12_1@PAGE1162; CHECK-NEXT: Lloh14:1163; CHECK-NEXT: adrp x10, lCPI12_2@PAGE1164; CHECK-NEXT: Lloh15:1165; CHECK-NEXT: ldr q0, [x8, lCPI12_0@PAGEOFF]1166; CHECK-NEXT: Lloh16:1167; CHECK-NEXT: ldr q1, [x9, lCPI12_1@PAGEOFF]1168; CHECK-NEXT: Lloh17:1169; CHECK-NEXT: ldr q2, [x10, lCPI12_2@PAGEOFF]1170; CHECK-NEXT: mov x8, xzr1171; CHECK-NEXT: LBB12_1: ; %loop1172; CHECK-NEXT: ; =>This Inner Loop Header: Depth=11173; CHECK-NEXT: ldr q3, [x0, x8]1174; CHECK-NEXT: add x8, x8, #161175; CHECK-NEXT: cmp x8, #1281176; CHECK-NEXT: tbl.16b v4, { v3 }, v21177; CHECK-NEXT: tbl.16b v5, { v3 }, v11178; CHECK-NEXT: tbl.16b v3, { v3 }, v01179; CHECK-NEXT: stp q5, q4, [x1, #16]1180; CHECK-NEXT: str q3, [x1], #641181; CHECK-NEXT: b.ne LBB12_11182; CHECK-NEXT: ; %bb.2: ; %exit1183; CHECK-NEXT: ret1184; CHECK-NEXT: .loh AdrpLdr Lloh14, Lloh171185; CHECK-NEXT: .loh AdrpLdr Lloh13, Lloh161186; CHECK-NEXT: .loh AdrpLdr Lloh12, Lloh151187;1188; CHECK-BE-LABEL: zext_v12i8_to_v12i32_in_loop:1189; CHECK-BE: // %bb.0: // %entry1190; CHECK-BE-NEXT: adrp x8, .LCPI12_01191; CHECK-BE-NEXT: add x8, x8, :lo12:.LCPI12_01192; CHECK-BE-NEXT: ld1 { v0.16b }, [x8]1193; CHECK-BE-NEXT: adrp x8, .LCPI12_11194; CHECK-BE-NEXT: add x8, x8, :lo12:.LCPI12_11195; CHECK-BE-NEXT: ld1 { v1.16b }, [x8]1196; CHECK-BE-NEXT: adrp x8, .LCPI12_21197; CHECK-BE-NEXT: add x8, x8, :lo12:.LCPI12_21198; CHECK-BE-NEXT: ld1 { v2.16b }, [x8]1199; CHECK-BE-NEXT: mov x8, xzr1200; CHECK-BE-NEXT: .LBB12_1: // %loop1201; CHECK-BE-NEXT: // =>This Inner Loop Header: Depth=11202; CHECK-BE-NEXT: add x9, x0, x81203; CHECK-BE-NEXT: add x8, x8, #161204; CHECK-BE-NEXT: add x10, x1, #161205; CHECK-BE-NEXT: ld1 { v3.16b }, [x9]1206; CHECK-BE-NEXT: add x9, x1, #321207; CHECK-BE-NEXT: cmp x8, #1281208; CHECK-BE-NEXT: tbl v4.16b, { v3.16b }, v2.16b1209; CHECK-BE-NEXT: tbl v5.16b, { v3.16b }, v1.16b1210; CHECK-BE-NEXT: tbl v3.16b, { v3.16b }, v0.16b1211; CHECK-BE-NEXT: st1 { v3.16b }, [x1]1212; CHECK-BE-NEXT: add x1, x1, #641213; CHECK-BE-NEXT: st1 { v4.16b }, [x9]1214; CHECK-BE-NEXT: st1 { v5.16b }, [x10]1215; CHECK-BE-NEXT: b.ne .LBB12_11216; CHECK-BE-NEXT: // %bb.2: // %exit1217; CHECK-BE-NEXT: ret1218 1219 1220entry:1221 br label %loop1222 1223loop:1224 %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]1225 %src.gep = getelementptr i8, ptr %src, i64 %iv1226 %load = load <12 x i8>, ptr %src.gep1227 %ext = zext <12 x i8> %load to <12 x i32>1228 %dst.gep = getelementptr i32, ptr %dst, i64 %iv1229 store <12 x i32> %ext, ptr %dst.gep1230 %iv.next = add nuw i64 %iv, 161231 %ec = icmp eq i64 %iv.next, 1281232 br i1 %ec, label %exit, label %loop1233 1234exit:1235 ret void1236}1237 1238define void @zext_v16i4_to_v16i32_in_loop(ptr %src, ptr %dst) {1239; CHECK-LABEL: zext_v16i4_to_v16i32_in_loop:1240; CHECK: ; %bb.0: ; %entry1241; CHECK-NEXT: movi.4s v0, #151242; CHECK-NEXT: mov x8, xzr1243; CHECK-NEXT: LBB13_1: ; %loop1244; CHECK-NEXT: ; =>This Inner Loop Header: Depth=11245; CHECK-NEXT: ldr x9, [x0, x8]1246; CHECK-NEXT: add x8, x8, #161247; CHECK-NEXT: cmp x8, #1281248; CHECK-NEXT: ubfx x12, x9, #48, #41249; CHECK-NEXT: lsr x10, x9, #521250; CHECK-NEXT: ubfx x13, x9, #32, #41251; CHECK-NEXT: ubfx w15, w9, #16, #41252; CHECK-NEXT: lsr x11, x9, #361253; CHECK-NEXT: lsr w14, w9, #201254; CHECK-NEXT: fmov s1, w121255; CHECK-NEXT: fmov s2, w131256; CHECK-NEXT: lsr w12, w9, #41257; CHECK-NEXT: fmov s3, w151258; CHECK-NEXT: mov.h v1[1], w101259; CHECK-NEXT: and w10, w9, #0xf1260; CHECK-NEXT: mov.h v2[1], w111261; CHECK-NEXT: fmov s4, w101262; CHECK-NEXT: lsr x11, x9, #561263; CHECK-NEXT: mov.h v3[1], w141264; CHECK-NEXT: lsr x10, x9, #401265; CHECK-NEXT: mov.h v4[1], w121266; CHECK-NEXT: lsr w12, w9, #241267; CHECK-NEXT: mov.h v1[2], w111268; CHECK-NEXT: lsr w11, w9, #81269; CHECK-NEXT: mov.h v2[2], w101270; CHECK-NEXT: lsr x10, x9, #601271; CHECK-NEXT: mov.h v3[2], w121272; CHECK-NEXT: lsr x12, x9, #441273; CHECK-NEXT: mov.h v4[2], w111274; CHECK-NEXT: lsr w11, w9, #281275; CHECK-NEXT: lsr w9, w9, #121276; CHECK-NEXT: mov.h v1[3], w101277; CHECK-NEXT: mov.h v2[3], w121278; CHECK-NEXT: mov.h v3[3], w111279; CHECK-NEXT: mov.h v4[3], w91280; CHECK-NEXT: ushll.4s v1, v1, #01281; CHECK-NEXT: ushll.4s v2, v2, #01282; CHECK-NEXT: ushll.4s v3, v3, #01283; CHECK-NEXT: ushll.4s v4, v4, #01284; CHECK-NEXT: and.16b v1, v1, v01285; CHECK-NEXT: and.16b v2, v2, v01286; CHECK-NEXT: and.16b v3, v3, v01287; CHECK-NEXT: and.16b v4, v4, v01288; CHECK-NEXT: stp q2, q1, [x1, #32]1289; CHECK-NEXT: stp q4, q3, [x1], #641290; CHECK-NEXT: b.ne LBB13_11291; CHECK-NEXT: ; %bb.2: ; %exit1292; CHECK-NEXT: ret1293;1294; CHECK-BE-LABEL: zext_v16i4_to_v16i32_in_loop:1295; CHECK-BE: // %bb.0: // %entry1296; CHECK-BE-NEXT: movi v0.4s, #151297; CHECK-BE-NEXT: mov x8, xzr1298; CHECK-BE-NEXT: .LBB13_1: // %loop1299; CHECK-BE-NEXT: // =>This Inner Loop Header: Depth=11300; CHECK-BE-NEXT: ldr x9, [x0, x8]1301; CHECK-BE-NEXT: add x8, x8, #161302; CHECK-BE-NEXT: cmp x8, #1281303; CHECK-BE-NEXT: ubfx w11, w9, #12, #41304; CHECK-BE-NEXT: lsr w14, w9, #281305; CHECK-BE-NEXT: lsr w10, w9, #81306; CHECK-BE-NEXT: ubfx x15, x9, #44, #41307; CHECK-BE-NEXT: lsr w12, w9, #241308; CHECK-BE-NEXT: lsr x13, x9, #401309; CHECK-BE-NEXT: fmov s1, w111310; CHECK-BE-NEXT: lsr x11, x9, #601311; CHECK-BE-NEXT: fmov s2, w141312; CHECK-BE-NEXT: fmov s3, w151313; CHECK-BE-NEXT: fmov s4, w111314; CHECK-BE-NEXT: lsr w11, w9, #201315; CHECK-BE-NEXT: mov v1.h[1], w101316; CHECK-BE-NEXT: lsr x10, x9, #561317; CHECK-BE-NEXT: mov v2.h[1], w121318; CHECK-BE-NEXT: lsr w12, w9, #41319; CHECK-BE-NEXT: mov v3.h[1], w131320; CHECK-BE-NEXT: mov v4.h[1], w101321; CHECK-BE-NEXT: lsr x10, x9, #361322; CHECK-BE-NEXT: mov v1.h[2], w121323; CHECK-BE-NEXT: lsr x12, x9, #521324; CHECK-BE-NEXT: mov v2.h[2], w111325; CHECK-BE-NEXT: mov v3.h[2], w101326; CHECK-BE-NEXT: lsr w10, w9, #161327; CHECK-BE-NEXT: lsr x11, x9, #321328; CHECK-BE-NEXT: mov v4.h[2], w121329; CHECK-BE-NEXT: mov v1.h[3], w91330; CHECK-BE-NEXT: lsr x9, x9, #481331; CHECK-BE-NEXT: mov v2.h[3], w101332; CHECK-BE-NEXT: mov v3.h[3], w111333; CHECK-BE-NEXT: add x10, x1, #321334; CHECK-BE-NEXT: mov v4.h[3], w91335; CHECK-BE-NEXT: add x9, x1, #481336; CHECK-BE-NEXT: ushll v1.4s, v1.4h, #01337; CHECK-BE-NEXT: ushll v2.4s, v2.4h, #01338; CHECK-BE-NEXT: ushll v3.4s, v3.4h, #01339; CHECK-BE-NEXT: ushll v4.4s, v4.4h, #01340; CHECK-BE-NEXT: and v1.16b, v1.16b, v0.16b1341; CHECK-BE-NEXT: and v2.16b, v2.16b, v0.16b1342; CHECK-BE-NEXT: and v3.16b, v3.16b, v0.16b1343; CHECK-BE-NEXT: and v4.16b, v4.16b, v0.16b1344; CHECK-BE-NEXT: st1 { v1.4s }, [x9]1345; CHECK-BE-NEXT: add x9, x1, #161346; CHECK-BE-NEXT: st1 { v2.4s }, [x10]1347; CHECK-BE-NEXT: st1 { v4.4s }, [x1]1348; CHECK-BE-NEXT: add x1, x1, #641349; CHECK-BE-NEXT: st1 { v3.4s }, [x9]1350; CHECK-BE-NEXT: b.ne .LBB13_11351; CHECK-BE-NEXT: // %bb.2: // %exit1352; CHECK-BE-NEXT: ret1353 1354 1355entry:1356 br label %loop1357 1358loop:1359 %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]1360 %src.gep = getelementptr i4, ptr %src, i64 %iv1361 %load = load <16 x i4>, ptr %src.gep1362 %ext = zext <16 x i4> %load to <16 x i32>1363 %dst.gep = getelementptr i32, ptr %dst, i64 %iv1364 store <16 x i32> %ext, ptr %dst.gep1365 %iv.next = add nuw i64 %iv, 161366 %ec = icmp eq i64 %iv.next, 1281367 br i1 %ec, label %exit, label %loop1368 1369exit:1370 ret void1371}1372 1373define void @zext_v16i16_to_v16i64_in_loop(ptr %src, ptr %dst) {1374; CHECK-LABEL: zext_v16i16_to_v16i64_in_loop:1375; CHECK: ; %bb.0: ; %entry1376; CHECK-NEXT: mov x8, xzr1377; CHECK-NEXT: LBB14_1: ; %loop1378; CHECK-NEXT: ; =>This Inner Loop Header: Depth=11379; CHECK-NEXT: add x9, x0, x81380; CHECK-NEXT: add x8, x8, #321381; CHECK-NEXT: ldp q1, q0, [x9]1382; CHECK-NEXT: cmp x8, #2561383; CHECK-NEXT: ushll2.4s v2, v0, #01384; CHECK-NEXT: ushll2.4s v3, v1, #01385; CHECK-NEXT: ushll.4s v0, v0, #01386; CHECK-NEXT: ushll.4s v1, v1, #01387; CHECK-NEXT: ushll2.2d v4, v2, #01388; CHECK-NEXT: ushll.2d v2, v2, #01389; CHECK-NEXT: ushll2.2d v5, v3, #01390; CHECK-NEXT: ushll.2d v3, v3, #01391; CHECK-NEXT: stp q2, q4, [x1, #96]1392; CHECK-NEXT: ushll2.2d v4, v0, #01393; CHECK-NEXT: ushll.2d v0, v0, #01394; CHECK-NEXT: ushll2.2d v2, v1, #01395; CHECK-NEXT: ushll.2d v1, v1, #01396; CHECK-NEXT: stp q3, q5, [x1, #32]1397; CHECK-NEXT: stp q0, q4, [x1, #64]1398; CHECK-NEXT: stp q1, q2, [x1], #1281399; CHECK-NEXT: b.ne LBB14_11400; CHECK-NEXT: ; %bb.2: ; %exit1401; CHECK-NEXT: ret1402;1403; CHECK-BE-LABEL: zext_v16i16_to_v16i64_in_loop:1404; CHECK-BE: // %bb.0: // %entry1405; CHECK-BE-NEXT: mov x8, xzr1406; CHECK-BE-NEXT: .LBB14_1: // %loop1407; CHECK-BE-NEXT: // =>This Inner Loop Header: Depth=11408; CHECK-BE-NEXT: add x9, x0, x81409; CHECK-BE-NEXT: add x8, x8, #321410; CHECK-BE-NEXT: ld1 { v0.8h }, [x9]1411; CHECK-BE-NEXT: add x9, x9, #161412; CHECK-BE-NEXT: cmp x8, #2561413; CHECK-BE-NEXT: ld1 { v1.8h }, [x9]1414; CHECK-BE-NEXT: add x9, x1, #481415; CHECK-BE-NEXT: ushll2 v2.4s, v0.8h, #01416; CHECK-BE-NEXT: ushll v0.4s, v0.4h, #01417; CHECK-BE-NEXT: ushll2 v3.4s, v1.8h, #01418; CHECK-BE-NEXT: ushll v1.4s, v1.4h, #01419; CHECK-BE-NEXT: ushll2 v4.2d, v2.4s, #01420; CHECK-BE-NEXT: ushll v2.2d, v2.2s, #01421; CHECK-BE-NEXT: ushll2 v5.2d, v0.4s, #01422; CHECK-BE-NEXT: ushll v0.2d, v0.2s, #01423; CHECK-BE-NEXT: st1 { v4.2d }, [x9]1424; CHECK-BE-NEXT: add x9, x1, #321425; CHECK-BE-NEXT: ushll2 v4.2d, v3.4s, #01426; CHECK-BE-NEXT: st1 { v2.2d }, [x9]1427; CHECK-BE-NEXT: add x9, x1, #161428; CHECK-BE-NEXT: ushll v2.2d, v3.2s, #01429; CHECK-BE-NEXT: st1 { v5.2d }, [x9]1430; CHECK-BE-NEXT: add x9, x1, #1121431; CHECK-BE-NEXT: ushll2 v3.2d, v1.4s, #01432; CHECK-BE-NEXT: st1 { v4.2d }, [x9]1433; CHECK-BE-NEXT: add x9, x1, #961434; CHECK-BE-NEXT: ushll v1.2d, v1.2s, #01435; CHECK-BE-NEXT: st1 { v2.2d }, [x9]1436; CHECK-BE-NEXT: add x9, x1, #801437; CHECK-BE-NEXT: st1 { v3.2d }, [x9]1438; CHECK-BE-NEXT: add x9, x1, #641439; CHECK-BE-NEXT: st1 { v0.2d }, [x1]1440; CHECK-BE-NEXT: add x1, x1, #1281441; CHECK-BE-NEXT: st1 { v1.2d }, [x9]1442; CHECK-BE-NEXT: b.ne .LBB14_11443; CHECK-BE-NEXT: // %bb.2: // %exit1444; CHECK-BE-NEXT: ret1445 1446 1447entry:1448 br label %loop1449 1450loop:1451 %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]1452 %src.gep = getelementptr i16, ptr %src, i64 %iv1453 %load = load <16 x i16>, ptr %src.gep1454 %ext = zext <16 x i16> %load to <16 x i64>1455 %dst.gep = getelementptr i64, ptr %dst, i64 %iv1456 store <16 x i64> %ext, ptr %dst.gep1457 %iv.next = add nuw i64 %iv, 161458 %ec = icmp eq i64 %iv.next, 1281459 br i1 %ec, label %exit, label %loop1460 1461exit:1462 ret void1463}1464 1465define void @zext_v16i32_to_v16i64_in_loop(ptr %src, ptr %dst) {1466; CHECK-LABEL: zext_v16i32_to_v16i64_in_loop:1467; CHECK: ; %bb.0: ; %entry1468; CHECK-NEXT: mov x8, xzr1469; CHECK-NEXT: LBB15_1: ; %loop1470; CHECK-NEXT: ; =>This Inner Loop Header: Depth=11471; CHECK-NEXT: add x9, x0, x81472; CHECK-NEXT: add x8, x8, #641473; CHECK-NEXT: ldp q1, q0, [x9, #32]1474; CHECK-NEXT: cmp x8, #5121475; CHECK-NEXT: ldp q5, q4, [x9]1476; CHECK-NEXT: ushll2.2d v2, v0, #01477; CHECK-NEXT: ushll2.2d v3, v1, #01478; CHECK-NEXT: ushll.2d v0, v0, #01479; CHECK-NEXT: ushll.2d v1, v1, #01480; CHECK-NEXT: stp q0, q2, [x1, #96]1481; CHECK-NEXT: ushll2.2d v2, v4, #01482; CHECK-NEXT: ushll2.2d v0, v5, #01483; CHECK-NEXT: stp q1, q3, [x1, #64]1484; CHECK-NEXT: ushll.2d v3, v4, #01485; CHECK-NEXT: ushll.2d v1, v5, #01486; CHECK-NEXT: stp q3, q2, [x1, #32]1487; CHECK-NEXT: stp q1, q0, [x1], #1281488; CHECK-NEXT: b.ne LBB15_11489; CHECK-NEXT: ; %bb.2: ; %exit1490; CHECK-NEXT: ret1491;1492; CHECK-BE-LABEL: zext_v16i32_to_v16i64_in_loop:1493; CHECK-BE: // %bb.0: // %entry1494; CHECK-BE-NEXT: mov x8, xzr1495; CHECK-BE-NEXT: .LBB15_1: // %loop1496; CHECK-BE-NEXT: // =>This Inner Loop Header: Depth=11497; CHECK-BE-NEXT: add x9, x0, x81498; CHECK-BE-NEXT: add x8, x8, #641499; CHECK-BE-NEXT: ld1 { v0.4s }, [x9]1500; CHECK-BE-NEXT: add x10, x9, #481501; CHECK-BE-NEXT: cmp x8, #5121502; CHECK-BE-NEXT: ld1 { v1.4s }, [x10]1503; CHECK-BE-NEXT: add x10, x9, #321504; CHECK-BE-NEXT: add x9, x9, #161505; CHECK-BE-NEXT: ld1 { v4.4s }, [x9]1506; CHECK-BE-NEXT: ld1 { v2.4s }, [x10]1507; CHECK-BE-NEXT: add x9, x1, #161508; CHECK-BE-NEXT: ushll2 v3.2d, v0.4s, #01509; CHECK-BE-NEXT: ushll v0.2d, v0.2s, #01510; CHECK-BE-NEXT: add x10, x1, #801511; CHECK-BE-NEXT: ushll2 v5.2d, v1.4s, #01512; CHECK-BE-NEXT: ushll2 v6.2d, v2.4s, #01513; CHECK-BE-NEXT: st1 { v3.2d }, [x9]1514; CHECK-BE-NEXT: ushll2 v3.2d, v4.4s, #01515; CHECK-BE-NEXT: add x9, x1, #1121516; CHECK-BE-NEXT: st1 { v0.2d }, [x1]1517; CHECK-BE-NEXT: ushll v0.2d, v1.2s, #01518; CHECK-BE-NEXT: ushll v1.2d, v2.2s, #01519; CHECK-BE-NEXT: st1 { v5.2d }, [x9]1520; CHECK-BE-NEXT: add x9, x1, #481521; CHECK-BE-NEXT: ushll v2.2d, v4.2s, #01522; CHECK-BE-NEXT: st1 { v3.2d }, [x9]1523; CHECK-BE-NEXT: add x9, x1, #641524; CHECK-BE-NEXT: st1 { v6.2d }, [x10]1525; CHECK-BE-NEXT: add x10, x1, #961526; CHECK-BE-NEXT: st1 { v1.2d }, [x9]1527; CHECK-BE-NEXT: add x9, x1, #321528; CHECK-BE-NEXT: add x1, x1, #1281529; CHECK-BE-NEXT: st1 { v0.2d }, [x10]1530; CHECK-BE-NEXT: st1 { v2.2d }, [x9]1531; CHECK-BE-NEXT: b.ne .LBB15_11532; CHECK-BE-NEXT: // %bb.2: // %exit1533; CHECK-BE-NEXT: ret1534 1535 1536entry:1537 br label %loop1538 1539loop:1540 %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]1541 %src.gep = getelementptr i32, ptr %src, i64 %iv1542 %load = load <16 x i32>, ptr %src.gep1543 %ext = zext <16 x i32> %load to <16 x i64>1544 %dst.gep = getelementptr i64, ptr %dst, i64 %iv1545 store <16 x i64> %ext, ptr %dst.gep1546 %iv.next = add nuw i64 %iv, 161547 %ec = icmp eq i64 %iv.next, 1281548 br i1 %ec, label %exit, label %loop1549 1550exit:1551 ret void1552}1553 1554define void @zext_v8i8_to_v8i128_in_loop(ptr %src, ptr %dst) {1555; CHECK-LABEL: zext_v8i8_to_v8i128_in_loop:1556; CHECK: ; %bb.0: ; %entry1557; CHECK-NEXT: mov x8, xzr1558; CHECK-NEXT: LBB16_1: ; %loop1559; CHECK-NEXT: ; =>This Inner Loop Header: Depth=11560; CHECK-NEXT: ldr d0, [x0, x8]1561; CHECK-NEXT: add x8, x8, #161562; CHECK-NEXT: str xzr, [x1, #120]1563; CHECK-NEXT: str xzr, [x1, #104]1564; CHECK-NEXT: cmp x8, #1281565; CHECK-NEXT: ushll.8h v0, v0, #01566; CHECK-NEXT: str xzr, [x1, #88]1567; CHECK-NEXT: str xzr, [x1, #72]1568; CHECK-NEXT: str xzr, [x1, #56]1569; CHECK-NEXT: ushll2.4s v1, v0, #01570; CHECK-NEXT: ushll.4s v0, v0, #01571; CHECK-NEXT: str xzr, [x1, #40]1572; CHECK-NEXT: str xzr, [x1, #24]1573; CHECK-NEXT: str xzr, [x1, #8]1574; CHECK-NEXT: ushll2.2d v2, v1, #01575; CHECK-NEXT: ushll.2d v1, v1, #01576; CHECK-NEXT: ushll2.2d v3, v0, #01577; CHECK-NEXT: ushll.2d v0, v0, #01578; CHECK-NEXT: str d2, [x1, #96]1579; CHECK-NEXT: mov d2, v2[1]1580; CHECK-NEXT: str d1, [x1, #64]1581; CHECK-NEXT: mov d1, v1[1]1582; CHECK-NEXT: str d3, [x1, #32]1583; CHECK-NEXT: mov d3, v3[1]1584; CHECK-NEXT: str d0, [x1]1585; CHECK-NEXT: mov d0, v0[1]1586; CHECK-NEXT: str d2, [x1, #112]1587; CHECK-NEXT: str d1, [x1, #80]1588; CHECK-NEXT: str d3, [x1, #48]1589; CHECK-NEXT: str d0, [x1, #16]1590; CHECK-NEXT: add x1, x1, #2561591; CHECK-NEXT: b.ne LBB16_11592; CHECK-NEXT: ; %bb.2: ; %exit1593; CHECK-NEXT: ret1594;1595; CHECK-BE-LABEL: zext_v8i8_to_v8i128_in_loop:1596; CHECK-BE: // %bb.0: // %entry1597; CHECK-BE-NEXT: mov x8, xzr1598; CHECK-BE-NEXT: .LBB16_1: // %loop1599; CHECK-BE-NEXT: // =>This Inner Loop Header: Depth=11600; CHECK-BE-NEXT: add x9, x0, x81601; CHECK-BE-NEXT: add x8, x8, #161602; CHECK-BE-NEXT: ld1 { v0.8b }, [x9]1603; CHECK-BE-NEXT: str xzr, [x1, #112]1604; CHECK-BE-NEXT: cmp x8, #1281605; CHECK-BE-NEXT: str xzr, [x1, #96]1606; CHECK-BE-NEXT: str xzr, [x1, #80]1607; CHECK-BE-NEXT: ushll v0.8h, v0.8b, #01608; CHECK-BE-NEXT: str xzr, [x1, #64]1609; CHECK-BE-NEXT: str xzr, [x1, #48]1610; CHECK-BE-NEXT: str xzr, [x1, #32]1611; CHECK-BE-NEXT: ushll2 v1.4s, v0.8h, #01612; CHECK-BE-NEXT: ushll v0.4s, v0.4h, #01613; CHECK-BE-NEXT: str xzr, [x1, #16]1614; CHECK-BE-NEXT: str xzr, [x1]1615; CHECK-BE-NEXT: ushll2 v2.2d, v1.4s, #01616; CHECK-BE-NEXT: ushll v1.2d, v1.2s, #01617; CHECK-BE-NEXT: ushll2 v3.2d, v0.4s, #01618; CHECK-BE-NEXT: ushll v0.2d, v0.2s, #01619; CHECK-BE-NEXT: str d2, [x1, #104]1620; CHECK-BE-NEXT: mov d2, v2.d[1]1621; CHECK-BE-NEXT: str d1, [x1, #72]1622; CHECK-BE-NEXT: mov d1, v1.d[1]1623; CHECK-BE-NEXT: str d3, [x1, #40]1624; CHECK-BE-NEXT: mov d3, v3.d[1]1625; CHECK-BE-NEXT: str d0, [x1, #8]1626; CHECK-BE-NEXT: mov d0, v0.d[1]1627; CHECK-BE-NEXT: str d2, [x1, #120]1628; CHECK-BE-NEXT: str d1, [x1, #88]1629; CHECK-BE-NEXT: str d3, [x1, #56]1630; CHECK-BE-NEXT: str d0, [x1, #24]1631; CHECK-BE-NEXT: add x1, x1, #2561632; CHECK-BE-NEXT: b.ne .LBB16_11633; CHECK-BE-NEXT: // %bb.2: // %exit1634; CHECK-BE-NEXT: ret1635 1636 1637entry:1638 br label %loop1639 1640loop:1641 %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]1642 %src.gep = getelementptr i8, ptr %src, i64 %iv1643 %load = load <8 x i8>, ptr %src.gep1644 %ext = zext <8 x i8> %load to <8 x i128>1645 %dst.gep = getelementptr i128, ptr %dst, i64 %iv1646 store <8 x i128> %ext, ptr %dst.gep1647 %iv.next = add nuw i64 %iv, 161648 %ec = icmp eq i64 %iv.next, 1281649 br i1 %ec, label %exit, label %loop1650 1651exit:1652 ret void1653}1654 1655; multiple back-to-back 'zext' of similar type of vectors combined with arithmetic operations1656define void @zext_v8i8_to_v8i64_with_add_in_sequence_in_loop(ptr %src, ptr %dst) {1657; CHECK-LABEL: zext_v8i8_to_v8i64_with_add_in_sequence_in_loop:1658; CHECK: ; %bb.0: ; %entry1659; CHECK-NEXT: Lloh18:1660; CHECK-NEXT: adrp x8, lCPI17_0@PAGE1661; CHECK-NEXT: Lloh19:1662; CHECK-NEXT: adrp x9, lCPI17_1@PAGE1663; CHECK-NEXT: mov w10, #128 ; =0x801664; CHECK-NEXT: Lloh20:1665; CHECK-NEXT: ldr q0, [x8, lCPI17_0@PAGEOFF]1666; CHECK-NEXT: Lloh21:1667; CHECK-NEXT: ldr q1, [x9, lCPI17_1@PAGEOFF]1668; CHECK-NEXT: add x8, x1, #641669; CHECK-NEXT: add x9, x0, #81670; CHECK-NEXT: LBB17_1: ; %loop1671; CHECK-NEXT: ; =>This Inner Loop Header: Depth=11672; CHECK-NEXT: ldp d2, d3, [x9, #-8]1673; CHECK-NEXT: subs x10, x10, #161674; CHECK-NEXT: ldp q7, q5, [x8, #-32]1675; CHECK-NEXT: add x9, x9, #161676; CHECK-NEXT: ldp q17, q6, [x8, #-64]1677; CHECK-NEXT: tbl.16b v4, { v2 }, v11678; CHECK-NEXT: tbl.16b v2, { v2 }, v01679; CHECK-NEXT: tbl.16b v16, { v3 }, v11680; CHECK-NEXT: tbl.16b v3, { v3 }, v01681; CHECK-NEXT: uaddw2.2d v5, v5, v41682; CHECK-NEXT: uaddw2.2d v6, v6, v21683; CHECK-NEXT: uaddw.2d v4, v7, v41684; CHECK-NEXT: ldp q18, q7, [x8, #32]1685; CHECK-NEXT: uaddw.2d v2, v17, v21686; CHECK-NEXT: stp q4, q5, [x8, #-32]1687; CHECK-NEXT: uaddw2.2d v5, v7, v161688; CHECK-NEXT: stp q2, q6, [x8, #-64]1689; CHECK-NEXT: uaddw.2d v16, v18, v161690; CHECK-NEXT: ldp q7, q6, [x8]1691; CHECK-NEXT: stp q16, q5, [x8, #32]1692; CHECK-NEXT: uaddw2.2d v4, v6, v31693; CHECK-NEXT: uaddw.2d v2, v7, v31694; CHECK-NEXT: stp q2, q4, [x8], #1281695; CHECK-NEXT: b.ne LBB17_11696; CHECK-NEXT: ; %bb.2: ; %exit1697; CHECK-NEXT: ret1698; CHECK-NEXT: .loh AdrpLdr Lloh19, Lloh211699; CHECK-NEXT: .loh AdrpLdr Lloh18, Lloh201700;1701; CHECK-BE-LABEL: zext_v8i8_to_v8i64_with_add_in_sequence_in_loop:1702; CHECK-BE: // %bb.0: // %entry1703; CHECK-BE-NEXT: adrp x9, .LCPI17_01704; CHECK-BE-NEXT: add x9, x9, :lo12:.LCPI17_01705; CHECK-BE-NEXT: mov w8, #128 // =0x801706; CHECK-BE-NEXT: ld1 { v0.16b }, [x9]1707; CHECK-BE-NEXT: adrp x9, .LCPI17_11708; CHECK-BE-NEXT: add x9, x9, :lo12:.LCPI17_11709; CHECK-BE-NEXT: ld1 { v1.16b }, [x9]1710; CHECK-BE-NEXT: add x9, x1, #641711; CHECK-BE-NEXT: add x10, x0, #81712; CHECK-BE-NEXT: .LBB17_1: // %loop1713; CHECK-BE-NEXT: // =>This Inner Loop Header: Depth=11714; CHECK-BE-NEXT: ld1 { v2.8b }, [x10]1715; CHECK-BE-NEXT: sub x11, x10, #81716; CHECK-BE-NEXT: add x15, x9, #321717; CHECK-BE-NEXT: ld1 { v3.8b }, [x11]1718; CHECK-BE-NEXT: ld1 { v16.2d }, [x15]1719; CHECK-BE-NEXT: sub x11, x9, #641720; CHECK-BE-NEXT: sub x12, x9, #321721; CHECK-BE-NEXT: ld1 { v6.2d }, [x9]1722; CHECK-BE-NEXT: ld1 { v21.2d }, [x11]1723; CHECK-BE-NEXT: tbl v4.16b, { v2.16b }, v1.16b1724; CHECK-BE-NEXT: tbl v2.16b, { v2.16b }, v0.16b1725; CHECK-BE-NEXT: ld1 { v19.2d }, [x12]1726; CHECK-BE-NEXT: tbl v5.16b, { v3.16b }, v1.16b1727; CHECK-BE-NEXT: tbl v3.16b, { v3.16b }, v0.16b1728; CHECK-BE-NEXT: sub x13, x9, #161729; CHECK-BE-NEXT: sub x14, x9, #481730; CHECK-BE-NEXT: add x16, x9, #481731; CHECK-BE-NEXT: add x17, x9, #161732; CHECK-BE-NEXT: ld1 { v22.2d }, [x13]1733; CHECK-BE-NEXT: subs x8, x8, #161734; CHECK-BE-NEXT: add x10, x10, #161735; CHECK-BE-NEXT: rev32 v7.8b, v4.8b1736; CHECK-BE-NEXT: ext v4.16b, v4.16b, v4.16b, #81737; CHECK-BE-NEXT: rev32 v17.8b, v2.8b1738; CHECK-BE-NEXT: ext v18.16b, v5.16b, v5.16b, #81739; CHECK-BE-NEXT: ext v20.16b, v3.16b, v3.16b, #81740; CHECK-BE-NEXT: ext v2.16b, v2.16b, v2.16b, #81741; CHECK-BE-NEXT: rev32 v5.8b, v5.8b1742; CHECK-BE-NEXT: rev32 v3.8b, v3.8b1743; CHECK-BE-NEXT: uaddw v7.2d, v16.2d, v7.2s1744; CHECK-BE-NEXT: rev32 v4.8b, v4.8b1745; CHECK-BE-NEXT: uaddw v6.2d, v6.2d, v17.2s1746; CHECK-BE-NEXT: rev32 v17.8b, v18.8b1747; CHECK-BE-NEXT: rev32 v20.8b, v20.8b1748; CHECK-BE-NEXT: rev32 v2.8b, v2.8b1749; CHECK-BE-NEXT: ld1 { v16.2d }, [x16]1750; CHECK-BE-NEXT: ld1 { v18.2d }, [x14]1751; CHECK-BE-NEXT: uaddw v5.2d, v19.2d, v5.2s1752; CHECK-BE-NEXT: uaddw v3.2d, v21.2d, v3.2s1753; CHECK-BE-NEXT: st1 { v7.2d }, [x15]1754; CHECK-BE-NEXT: ld1 { v7.2d }, [x17]1755; CHECK-BE-NEXT: st1 { v6.2d }, [x9]1756; CHECK-BE-NEXT: add x9, x9, #1281757; CHECK-BE-NEXT: uaddw v4.2d, v16.2d, v4.2s1758; CHECK-BE-NEXT: st1 { v5.2d }, [x12]1759; CHECK-BE-NEXT: uaddw v5.2d, v22.2d, v17.2s1760; CHECK-BE-NEXT: st1 { v3.2d }, [x11]1761; CHECK-BE-NEXT: uaddw v3.2d, v18.2d, v20.2s1762; CHECK-BE-NEXT: uaddw v2.2d, v7.2d, v2.2s1763; CHECK-BE-NEXT: st1 { v4.2d }, [x16]1764; CHECK-BE-NEXT: st1 { v5.2d }, [x13]1765; CHECK-BE-NEXT: st1 { v3.2d }, [x14]1766; CHECK-BE-NEXT: st1 { v2.2d }, [x17]1767; CHECK-BE-NEXT: b.ne .LBB17_11768; CHECK-BE-NEXT: // %bb.2: // %exit1769; CHECK-BE-NEXT: ret1770 1771 1772entry:1773 br label %loop1774 1775loop:1776 %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]1777 %src.gep = getelementptr i8, ptr %src, i64 %iv1778 %load = load <8 x i8>, ptr %src.gep1779 %src.gep.2 = getelementptr i8, ptr %src.gep, i64 81780 %load.2 = load <8 x i8>, ptr %src.gep.21781 %ext = zext <8 x i8> %load to <8 x i64>1782 %ext.2 = zext <8 x i8> %load.2 to <8 x i64>1783 %dst.gep = getelementptr i64, ptr %dst, i64 %iv1784 %load.dst = load <8 x i64>, ptr %dst.gep1785 %dst.gep.2 = getelementptr i64, ptr %dst.gep, i64 81786 %load.dst.2 = load <8 x i64>, ptr %dst.gep.21787 %sum = add <8 x i64> %load.dst, %ext1788 %sum.2 = add <8 x i64> %load.dst.2, %ext.21789 store <8 x i64> %sum, ptr %dst.gep1790 store <8 x i64> %sum.2, ptr %dst.gep.21791 %iv.next = add nuw i64 %iv, 161792 %ec = icmp eq i64 %iv.next, 1281793 br i1 %ec, label %exit, label %loop1794 1795exit:1796 ret void1797}1798 1799; multiple back-to-back 'zext' of similar type of vectors1800define void @zext_v16i8_to_v16i64_in_sequence_in_loop(ptr %src, ptr %dst) {1801; CHECK-LABEL: zext_v16i8_to_v16i64_in_sequence_in_loop:1802; CHECK: ; %bb.0: ; %entry1803; CHECK-NEXT: mov w8, #128 ; =0x801804; CHECK-NEXT: add x9, x1, #1281805; CHECK-NEXT: add x10, x0, #161806; CHECK-NEXT: LBB18_1: ; %loop1807; CHECK-NEXT: ; =>This Inner Loop Header: Depth=11808; CHECK-NEXT: ldp q0, q1, [x10, #-16]1809; CHECK-NEXT: subs x8, x8, #161810; CHECK-NEXT: add x10, x10, #161811; CHECK-NEXT: ushll2.8h v2, v0, #01812; CHECK-NEXT: ushll.8h v0, v0, #01813; CHECK-NEXT: ushll2.8h v6, v1, #01814; CHECK-NEXT: ushll.8h v1, v1, #01815; CHECK-NEXT: ushll2.4s v3, v2, #01816; CHECK-NEXT: ushll.4s v2, v2, #01817; CHECK-NEXT: ushll2.4s v4, v0, #01818; CHECK-NEXT: ushll.4s v0, v0, #01819; CHECK-NEXT: ushll2.2d v5, v3, #01820; CHECK-NEXT: ushll.2d v3, v3, #01821; CHECK-NEXT: ushll2.2d v7, v2, #01822; CHECK-NEXT: ushll2.2d v16, v4, #01823; CHECK-NEXT: ushll.2d v2, v2, #01824; CHECK-NEXT: ushll.2d v4, v4, #01825; CHECK-NEXT: stp q3, q5, [x9, #-32]1826; CHECK-NEXT: ushll2.4s v3, v6, #01827; CHECK-NEXT: ushll2.2d v5, v0, #01828; CHECK-NEXT: ushll.2d v0, v0, #01829; CHECK-NEXT: stp q4, q16, [x9, #-96]1830; CHECK-NEXT: ushll.4s v6, v6, #01831; CHECK-NEXT: stp q2, q7, [x9, #-64]1832; CHECK-NEXT: ushll2.4s v4, v1, #01833; CHECK-NEXT: ushll2.2d v2, v3, #01834; CHECK-NEXT: ushll.2d v3, v3, #01835; CHECK-NEXT: stp q0, q5, [x9, #-128]1836; CHECK-NEXT: ushll.4s v0, v1, #01837; CHECK-NEXT: ushll2.2d v5, v6, #01838; CHECK-NEXT: ushll.2d v1, v6, #01839; CHECK-NEXT: stp q3, q2, [x9, #96]1840; CHECK-NEXT: ushll2.2d v2, v4, #01841; CHECK-NEXT: ushll.2d v3, v4, #01842; CHECK-NEXT: ushll2.2d v4, v0, #01843; CHECK-NEXT: ushll.2d v0, v0, #01844; CHECK-NEXT: stp q1, q5, [x9, #64]1845; CHECK-NEXT: stp q3, q2, [x9, #32]1846; CHECK-NEXT: stp q0, q4, [x9], #1281847; CHECK-NEXT: b.ne LBB18_11848; CHECK-NEXT: ; %bb.2: ; %exit1849; CHECK-NEXT: ret1850;1851; CHECK-BE-LABEL: zext_v16i8_to_v16i64_in_sequence_in_loop:1852; CHECK-BE: // %bb.0: // %entry1853; CHECK-BE-NEXT: mov w8, #128 // =0x801854; CHECK-BE-NEXT: add x9, x1, #1281855; CHECK-BE-NEXT: add x10, x0, #161856; CHECK-BE-NEXT: .LBB18_1: // %loop1857; CHECK-BE-NEXT: // =>This Inner Loop Header: Depth=11858; CHECK-BE-NEXT: sub x11, x10, #161859; CHECK-BE-NEXT: ld1 { v5.16b }, [x10]1860; CHECK-BE-NEXT: sub x12, x9, #321861; CHECK-BE-NEXT: ld1 { v0.16b }, [x11]1862; CHECK-BE-NEXT: sub x11, x9, #161863; CHECK-BE-NEXT: subs x8, x8, #161864; CHECK-BE-NEXT: add x10, x10, #161865; CHECK-BE-NEXT: ushll2 v1.8h, v0.16b, #01866; CHECK-BE-NEXT: ushll v0.8h, v0.8b, #01867; CHECK-BE-NEXT: ushll2 v2.4s, v1.8h, #01868; CHECK-BE-NEXT: ushll2 v3.4s, v0.8h, #01869; CHECK-BE-NEXT: ushll v1.4s, v1.4h, #01870; CHECK-BE-NEXT: ushll v0.4s, v0.4h, #01871; CHECK-BE-NEXT: ushll2 v4.2d, v2.4s, #01872; CHECK-BE-NEXT: ushll v2.2d, v2.2s, #01873; CHECK-BE-NEXT: ushll2 v6.2d, v1.4s, #01874; CHECK-BE-NEXT: ushll v1.2d, v1.2s, #01875; CHECK-BE-NEXT: st1 { v4.2d }, [x11]1876; CHECK-BE-NEXT: ushll2 v4.2d, v3.4s, #01877; CHECK-BE-NEXT: ushll v3.2d, v3.2s, #01878; CHECK-BE-NEXT: st1 { v2.2d }, [x12]1879; CHECK-BE-NEXT: ushll2 v2.8h, v5.16b, #01880; CHECK-BE-NEXT: sub x12, x9, #801881; CHECK-BE-NEXT: sub x11, x9, #481882; CHECK-BE-NEXT: st1 { v4.2d }, [x12]1883; CHECK-BE-NEXT: ushll v4.8h, v5.8b, #01884; CHECK-BE-NEXT: sub x12, x9, #641885; CHECK-BE-NEXT: ushll2 v5.4s, v2.8h, #01886; CHECK-BE-NEXT: st1 { v1.2d }, [x12]1887; CHECK-BE-NEXT: sub x12, x9, #961888; CHECK-BE-NEXT: ushll2 v1.2d, v0.4s, #01889; CHECK-BE-NEXT: ushll v2.4s, v2.4h, #01890; CHECK-BE-NEXT: ushll v0.2d, v0.2s, #01891; CHECK-BE-NEXT: st1 { v6.2d }, [x11]1892; CHECK-BE-NEXT: sub x11, x9, #1281893; CHECK-BE-NEXT: st1 { v3.2d }, [x12]1894; CHECK-BE-NEXT: ushll2 v3.4s, v4.8h, #01895; CHECK-BE-NEXT: ushll2 v6.2d, v5.4s, #01896; CHECK-BE-NEXT: sub x12, x9, #1121897; CHECK-BE-NEXT: ushll v5.2d, v5.2s, #01898; CHECK-BE-NEXT: st1 { v0.2d }, [x11]1899; CHECK-BE-NEXT: st1 { v1.2d }, [x12]1900; CHECK-BE-NEXT: ushll2 v1.2d, v2.4s, #01901; CHECK-BE-NEXT: add x11, x9, #1121902; CHECK-BE-NEXT: ushll v4.4s, v4.4h, #01903; CHECK-BE-NEXT: ushll2 v0.2d, v3.4s, #01904; CHECK-BE-NEXT: st1 { v6.2d }, [x11]1905; CHECK-BE-NEXT: add x11, x9, #961906; CHECK-BE-NEXT: ushll v2.2d, v2.2s, #01907; CHECK-BE-NEXT: ushll v3.2d, v3.2s, #01908; CHECK-BE-NEXT: st1 { v5.2d }, [x11]1909; CHECK-BE-NEXT: add x11, x9, #801910; CHECK-BE-NEXT: st1 { v1.2d }, [x11]1911; CHECK-BE-NEXT: add x11, x9, #481912; CHECK-BE-NEXT: ushll2 v1.2d, v4.4s, #01913; CHECK-BE-NEXT: st1 { v0.2d }, [x11]1914; CHECK-BE-NEXT: ushll v0.2d, v4.2s, #01915; CHECK-BE-NEXT: add x11, x9, #641916; CHECK-BE-NEXT: st1 { v2.2d }, [x11]1917; CHECK-BE-NEXT: add x11, x9, #321918; CHECK-BE-NEXT: st1 { v3.2d }, [x11]1919; CHECK-BE-NEXT: add x11, x9, #161920; CHECK-BE-NEXT: st1 { v0.2d }, [x9]1921; CHECK-BE-NEXT: add x9, x9, #1281922; CHECK-BE-NEXT: st1 { v1.2d }, [x11]1923; CHECK-BE-NEXT: b.ne .LBB18_11924; CHECK-BE-NEXT: // %bb.2: // %exit1925; CHECK-BE-NEXT: ret1926 1927 1928entry:1929 br label %loop1930 1931loop:1932 %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]1933 %src.gep = getelementptr i8, ptr %src, i64 %iv1934 %load = load <16 x i8>, ptr %src.gep1935 %src.gep.2 = getelementptr i8, ptr %src.gep, i64 161936 %load.2 = load <16 x i8>, ptr %src.gep.21937 %ext = zext <16 x i8> %load to <16 x i64>1938 %ext.2 = zext <16 x i8> %load.2 to <16 x i64>1939 %dst.gep = getelementptr i64, ptr %dst, i64 %iv1940 store <16 x i64> %ext, ptr %dst.gep1941 %dst.gep.2 = getelementptr i64, ptr %dst.gep, i64 161942 store <16 x i64> %ext.2, ptr %dst.gep.21943 %iv.next = add nuw i64 %iv, 161944 %ec = icmp eq i64 %iv.next, 1281945 br i1 %ec, label %exit, label %loop1946 1947exit:1948 ret void1949}1950 1951define void @zext_v16i8_to_v16i32_in_loop_scalable_vectors(ptr %src, ptr %dst) {1952; CHECK-LABEL: zext_v16i8_to_v16i32_in_loop_scalable_vectors:1953; CHECK: ; %bb.0: ; %entry1954; CHECK-NEXT: ptrue p0.s1955; CHECK-NEXT: mov x8, xzr1956; CHECK-NEXT: LBB19_1: ; %loop1957; CHECK-NEXT: ; =>This Inner Loop Header: Depth=11958; CHECK-NEXT: ld1b { z0.s }, p0/z, [x0, x8]1959; CHECK-NEXT: add x9, x0, x81960; CHECK-NEXT: ld1b { z1.s }, p0/z, [x9, #2, mul vl]1961; CHECK-NEXT: ld1b { z2.s }, p0/z, [x9, #3, mul vl]1962; CHECK-NEXT: ld1b { z3.s }, p0/z, [x9, #1, mul vl]1963; CHECK-NEXT: add x9, x1, x8, lsl #21964; CHECK-NEXT: add z0.s, z0.s, z0.s1965; CHECK-NEXT: add z1.s, z1.s, z1.s1966; CHECK-NEXT: add z2.s, z2.s, z2.s1967; CHECK-NEXT: add z3.s, z3.s, z3.s1968; CHECK-NEXT: st1w { z0.s }, p0, [x1, x8, lsl #2]1969; CHECK-NEXT: add x8, x8, #161970; CHECK-NEXT: cmp x8, #1281971; CHECK-NEXT: str z1, [x9, #2, mul vl]1972; CHECK-NEXT: str z2, [x9, #3, mul vl]1973; CHECK-NEXT: str z3, [x9, #1, mul vl]1974; CHECK-NEXT: b.ne LBB19_11975; CHECK-NEXT: ; %bb.2: ; %exit1976; CHECK-NEXT: ret1977;1978; CHECK-BE-LABEL: zext_v16i8_to_v16i32_in_loop_scalable_vectors:1979; CHECK-BE: // %bb.0: // %entry1980; CHECK-BE-NEXT: ptrue p0.s1981; CHECK-BE-NEXT: mov x8, xzr1982; CHECK-BE-NEXT: .LBB19_1: // %loop1983; CHECK-BE-NEXT: // =>This Inner Loop Header: Depth=11984; CHECK-BE-NEXT: ld1b { z0.s }, p0/z, [x0, x8]1985; CHECK-BE-NEXT: add x9, x0, x81986; CHECK-BE-NEXT: ld1b { z1.s }, p0/z, [x9, #2, mul vl]1987; CHECK-BE-NEXT: ld1b { z2.s }, p0/z, [x9, #3, mul vl]1988; CHECK-BE-NEXT: ld1b { z3.s }, p0/z, [x9, #1, mul vl]1989; CHECK-BE-NEXT: add x9, x1, x8, lsl #21990; CHECK-BE-NEXT: add z0.s, z0.s, z0.s1991; CHECK-BE-NEXT: add z1.s, z1.s, z1.s1992; CHECK-BE-NEXT: add z2.s, z2.s, z2.s1993; CHECK-BE-NEXT: add z3.s, z3.s, z3.s1994; CHECK-BE-NEXT: st1w { z0.s }, p0, [x1, x8, lsl #2]1995; CHECK-BE-NEXT: add x8, x8, #161996; CHECK-BE-NEXT: cmp x8, #1281997; CHECK-BE-NEXT: st1w { z1.s }, p0, [x9, #2, mul vl]1998; CHECK-BE-NEXT: st1w { z2.s }, p0, [x9, #3, mul vl]1999; CHECK-BE-NEXT: st1w { z3.s }, p0, [x9, #1, mul vl]2000; CHECK-BE-NEXT: b.ne .LBB19_12001; CHECK-BE-NEXT: // %bb.2: // %exit2002; CHECK-BE-NEXT: ret2003entry:2004 br label %loop2005 2006loop:2007 %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]2008 %src.gep = getelementptr i8, ptr %src, i64 %iv2009 %load = load <vscale x 16 x i8>, ptr %src.gep2010 %ext = zext <vscale x 16 x i8> %load to <vscale x 16 x i32>2011 %add = add <vscale x 16 x i32> %ext, %ext2012 %dst.gep = getelementptr i32, ptr %dst, i64 %iv2013 store <vscale x 16 x i32> %add, ptr %dst.gep2014 %iv.next = add nuw i64 %iv, 162015 %ec = icmp eq i64 %iv.next, 1282016 br i1 %ec, label %exit, label %loop2017 2018exit:2019 ret void2020}2021 2022; CHECK-LABEL: lCPI20_0:2023; CHECK-NEXT: .byte 0 ; 0x02024; CHECK-NEXT: .byte 255 ; 0xff2025; CHECK-NEXT: .byte 255 ; 0xff2026; CHECK-NEXT: .byte 1 ; 0x12027; CHECK-NEXT: .byte 255 ; 0xff2028; CHECK-NEXT: .byte 255 ; 0xff2029; CHECK-NEXT: .byte 2 ; 0x22030; CHECK-NEXT: .byte 255 ; 0xff2031; CHECK-NEXT: .byte 255 ; 0xff2032; CHECK-NEXT: .byte 3 ; 0x32033; CHECK-NEXT: .byte 255 ; 0xff2034; CHECK-NEXT: .byte 255 ; 0xff2035; CHECK-NEXT: .byte 255 ; 0xff2036; CHECK-NEXT: .byte 255 ; 0xff2037; CHECK-NEXT: .byte 255 ; 0xff2038; CHECK-NEXT: .byte 255 ; 0xff2039; CHECK-NEXT: lCPI20_1:2040; CHECK-NEXT: .byte 0 ; 0x02041; CHECK-NEXT: .byte 255 ; 0xff2042; CHECK-NEXT: .byte 255 ; 0xff2043; CHECK-NEXT: .byte 1 ; 0x12044; CHECK-NEXT: .byte 255 ; 0xff2045; CHECK-NEXT: .byte 255 ; 0xff2046; CHECK-NEXT: .byte 2 ; 0x22047; CHECK-NEXT: .byte 255 ; 0xff2048; CHECK-NEXT: .byte 255 ; 0xff2049; CHECK-NEXT: .byte 3 ; 0x32050; CHECK-NEXT: .byte 255 ; 0xff2051; CHECK-NEXT: .byte 255 ; 0xff2052; CHECK-NEXT: .byte 4 ; 0x42053; CHECK-NEXT: .byte 255 ; 0xff2054; CHECK-NEXT: .byte 255 ; 0xff2055; CHECK-NEXT: .byte 5 ; 0x52056; CHECK-NEXT: lCPI20_2:2057; CHECK-NEXT: .byte 255 ; 0xff2058; CHECK-NEXT: .byte 255 ; 0xff2059; CHECK-NEXT: .byte 6 ; 0x62060; CHECK-NEXT: .byte 255 ; 0xff2061; CHECK-NEXT: .byte 255 ; 0xff2062; CHECK-NEXT: .byte 7 ; 0x72063; CHECK-NEXT: .byte 255 ; 0xff2064; CHECK-NEXT: .byte 255 ; 0xff2065; CHECK-NEXT: .byte 8 ; 0x82066; CHECK-NEXT: .byte 255 ; 0xff2067; CHECK-NEXT: .byte 255 ; 0xff2068; CHECK-NEXT: .byte 9 ; 0x92069; CHECK-NEXT: .byte 255 ; 0xff2070; CHECK-NEXT: .byte 255 ; 0xff2071; CHECK-NEXT: .byte 10 ; 0xa2072; CHECK-NEXT: .byte 255 ; 0xff2073; CHECK-NEXT: lCPI20_3:2074; CHECK-NEXT: .byte 255 ; 0xff2075; CHECK-NEXT: .byte 11 ; 0xb2076; CHECK-NEXT: .byte 255 ; 0xff2077; CHECK-NEXT: .byte 255 ; 0xff2078; CHECK-NEXT: .byte 12 ; 0xc2079; CHECK-NEXT: .byte 255 ; 0xff2080; CHECK-NEXT: .byte 255 ; 0xff2081; CHECK-NEXT: .byte 13 ; 0xd2082; CHECK-NEXT: .byte 255 ; 0xff2083; CHECK-NEXT: .byte 255 ; 0xff2084; CHECK-NEXT: .byte 14 ; 0xe2085; CHECK-NEXT: .byte 255 ; 0xff2086; CHECK-NEXT: .byte 255 ; 0xff2087; CHECK-NEXT: .byte 15 ; 0xf2088; CHECK-NEXT: .byte 255 ; 0xff2089; CHECK-NEXT: .byte 255 ; 0xff2090 2091; CHECK-BE-LABEL: .LCPI20_0:2092; CHECK-BE-NEXT: .byte 255 // 0xff2093; CHECK-BE-NEXT: .byte 255 // 0xff2094; CHECK-BE-NEXT: .byte 0 // 0x02095; CHECK-BE-NEXT: .byte 255 // 0xff2096; CHECK-BE-NEXT: .byte 255 // 0xff2097; CHECK-BE-NEXT: .byte 1 // 0x12098; CHECK-BE-NEXT: .byte 255 // 0xff2099; CHECK-BE-NEXT: .byte 255 // 0xff2100; CHECK-BE-NEXT: .byte 2 // 0x22101; CHECK-BE-NEXT: .byte 255 // 0xff2102; CHECK-BE-NEXT: .byte 255 // 0xff2103; CHECK-BE-NEXT: .byte 3 // 0x32104; CHECK-BE-NEXT: .byte 255 // 0xff2105; CHECK-BE-NEXT: .byte 255 // 0xff2106; CHECK-BE-NEXT: .byte 255 // 0xff2107; CHECK-BE-NEXT: .byte 255 // 0xff2108; CHECK-BE-NEXT: .LCPI20_1:2109; CHECK-BE-NEXT: .byte 255 // 0xff2110; CHECK-BE-NEXT: .byte 255 // 0xff2111; CHECK-BE-NEXT: .byte 0 // 0x02112; CHECK-BE-NEXT: .byte 255 // 0xff2113; CHECK-BE-NEXT: .byte 255 // 0xff2114; CHECK-BE-NEXT: .byte 1 // 0x12115; CHECK-BE-NEXT: .byte 255 // 0xff2116; CHECK-BE-NEXT: .byte 255 // 0xff2117; CHECK-BE-NEXT: .byte 2 // 0x22118; CHECK-BE-NEXT: .byte 255 // 0xff2119; CHECK-BE-NEXT: .byte 255 // 0xff2120; CHECK-BE-NEXT: .byte 3 // 0x32121; CHECK-BE-NEXT: .byte 255 // 0xff2122; CHECK-BE-NEXT: .byte 255 // 0xff2123; CHECK-BE-NEXT: .byte 4 // 0x42124; CHECK-BE-NEXT: .byte 255 // 0xff2125; CHECK-BE-NEXT: .LCPI20_2:2126; CHECK-BE-NEXT: .byte 255 // 0xff2127; CHECK-BE-NEXT: .byte 5 // 0x52128; CHECK-BE-NEXT: .byte 255 // 0xff2129; CHECK-BE-NEXT: .byte 255 // 0xff2130; CHECK-BE-NEXT: .byte 6 // 0x62131; CHECK-BE-NEXT: .byte 255 // 0xff2132; CHECK-BE-NEXT: .byte 255 // 0xff2133; CHECK-BE-NEXT: .byte 7 // 0x72134; CHECK-BE-NEXT: .byte 255 // 0xff2135; CHECK-BE-NEXT: .byte 255 // 0xff2136; CHECK-BE-NEXT: .byte 8 // 0x82137; CHECK-BE-NEXT: .byte 255 // 0xff2138; CHECK-BE-NEXT: .byte 255 // 0xff2139; CHECK-BE-NEXT: .byte 9 // 0x92140; CHECK-BE-NEXT: .byte 255 // 0xff2141; CHECK-BE-NEXT: .byte 255 // 0xff2142; CHECK-BE-NEXT: .LCPI20_3:2143; CHECK-BE-NEXT: .byte 10 // 0xa2144; CHECK-BE-NEXT: .byte 255 // 0xff2145; CHECK-BE-NEXT: .byte 255 // 0xff2146; CHECK-BE-NEXT: .byte 11 // 0xb2147; CHECK-BE-NEXT: .byte 255 // 0xff2148; CHECK-BE-NEXT: .byte 255 // 0xff2149; CHECK-BE-NEXT: .byte 12 // 0xc2150; CHECK-BE-NEXT: .byte 255 // 0xff2151; CHECK-BE-NEXT: .byte 255 // 0xff2152; CHECK-BE-NEXT: .byte 13 // 0xd2153; CHECK-BE-NEXT: .byte 255 // 0xff2154; CHECK-BE-NEXT: .byte 255 // 0xff2155; CHECK-BE-NEXT: .byte 14 // 0xe2156; CHECK-BE-NEXT: .byte 255 // 0xff2157; CHECK-BE-NEXT: .byte 255 // 0xff2158; CHECK-BE-NEXT: .byte 15 // 0xf2159 2160define void @zext_v20i8_to_v20i24_in_loop(ptr %src, ptr %dst) {2161; CHECK-LABEL: zext_v20i8_to_v20i24_in_loop:2162; CHECK: ; %bb.0: ; %entry2163; CHECK-NEXT: Lloh22:2164; CHECK-NEXT: adrp x8, lCPI20_0@PAGE2165; CHECK-NEXT: Lloh23:2166; CHECK-NEXT: adrp x9, lCPI20_1@PAGE2167; CHECK-NEXT: Lloh24:2168; CHECK-NEXT: adrp x10, lCPI20_2@PAGE2169; CHECK-NEXT: Lloh25:2170; CHECK-NEXT: ldr q0, [x8, lCPI20_0@PAGEOFF]2171; CHECK-NEXT: Lloh26:2172; CHECK-NEXT: adrp x8, lCPI20_3@PAGE2173; CHECK-NEXT: Lloh27:2174; CHECK-NEXT: ldr q1, [x9, lCPI20_1@PAGEOFF]2175; CHECK-NEXT: Lloh28:2176; CHECK-NEXT: ldr q2, [x10, lCPI20_2@PAGEOFF]2177; CHECK-NEXT: Lloh29:2178; CHECK-NEXT: ldr q3, [x8, lCPI20_3@PAGEOFF]2179; CHECK-NEXT: mov x8, xzr2180; CHECK-NEXT: LBB20_1: ; %loop2181; CHECK-NEXT: ; =>This Inner Loop Header: Depth=12182; CHECK-NEXT: add x9, x0, x82183; CHECK-NEXT: add x8, x8, #162184; CHECK-NEXT: ldp q4, q5, [x9]2185; CHECK-NEXT: cmp x8, #1282186; CHECK-NEXT: tbl.16b v5, { v5 }, v02187; CHECK-NEXT: tbl.16b v6, { v4 }, v32188; CHECK-NEXT: tbl.16b v7, { v4 }, v22189; CHECK-NEXT: tbl.16b v4, { v4 }, v12190; CHECK-NEXT: stp q7, q6, [x1, #16]2191; CHECK-NEXT: mov s6, v5[2]2192; CHECK-NEXT: str q4, [x1]2193; CHECK-NEXT: str d5, [x1, #48]2194; CHECK-NEXT: str s6, [x1, #56]2195; CHECK-NEXT: add x1, x1, #642196; CHECK-NEXT: b.ne LBB20_12197; CHECK-NEXT: ; %bb.2: ; %exit2198; CHECK-NEXT: ret2199; CHECK-NEXT: .loh AdrpLdr Lloh26, Lloh292200; CHECK-NEXT: .loh AdrpLdr Lloh24, Lloh282201; CHECK-NEXT: .loh AdrpLdr Lloh23, Lloh272202; CHECK-NEXT: .loh AdrpAdrp Lloh22, Lloh262203; CHECK-NEXT: .loh AdrpLdr Lloh22, Lloh252204;2205; CHECK-BE-LABEL: zext_v20i8_to_v20i24_in_loop:2206; CHECK-BE: // %bb.0: // %entry2207; CHECK-BE-NEXT: adrp x8, .LCPI20_02208; CHECK-BE-NEXT: add x8, x8, :lo12:.LCPI20_02209; CHECK-BE-NEXT: ld1 { v0.16b }, [x8]2210; CHECK-BE-NEXT: adrp x8, .LCPI20_12211; CHECK-BE-NEXT: add x8, x8, :lo12:.LCPI20_12212; CHECK-BE-NEXT: ld1 { v1.16b }, [x8]2213; CHECK-BE-NEXT: adrp x8, .LCPI20_22214; CHECK-BE-NEXT: add x8, x8, :lo12:.LCPI20_22215; CHECK-BE-NEXT: ld1 { v2.16b }, [x8]2216; CHECK-BE-NEXT: adrp x8, .LCPI20_32217; CHECK-BE-NEXT: add x8, x8, :lo12:.LCPI20_32218; CHECK-BE-NEXT: ld1 { v3.16b }, [x8]2219; CHECK-BE-NEXT: mov x8, xzr2220; CHECK-BE-NEXT: .LBB20_1: // %loop2221; CHECK-BE-NEXT: // =>This Inner Loop Header: Depth=12222; CHECK-BE-NEXT: add x9, x0, x82223; CHECK-BE-NEXT: add x8, x8, #162224; CHECK-BE-NEXT: add x10, x9, #162225; CHECK-BE-NEXT: ld1 { v5.16b }, [x9]2226; CHECK-BE-NEXT: add x9, x1, #322227; CHECK-BE-NEXT: ld1 { v4.16b }, [x10]2228; CHECK-BE-NEXT: cmp x8, #1282229; CHECK-BE-NEXT: tbl v6.16b, { v5.16b }, v3.16b2230; CHECK-BE-NEXT: tbl v16.16b, { v5.16b }, v2.16b2231; CHECK-BE-NEXT: tbl v5.16b, { v5.16b }, v1.16b2232; CHECK-BE-NEXT: tbl v4.16b, { v4.16b }, v0.16b2233; CHECK-BE-NEXT: st1 { v6.16b }, [x9]2234; CHECK-BE-NEXT: add x9, x1, #162235; CHECK-BE-NEXT: rev32 v7.16b, v4.16b2236; CHECK-BE-NEXT: rev64 v4.16b, v4.16b2237; CHECK-BE-NEXT: st1 { v5.16b }, [x1]2238; CHECK-BE-NEXT: st1 { v16.16b }, [x9]2239; CHECK-BE-NEXT: mov s6, v7.s[2]2240; CHECK-BE-NEXT: str d4, [x1, #48]2241; CHECK-BE-NEXT: str s6, [x1, #56]2242; CHECK-BE-NEXT: add x1, x1, #642243; CHECK-BE-NEXT: b.ne .LBB20_12244; CHECK-BE-NEXT: // %bb.2: // %exit2245; CHECK-BE-NEXT: ret2246 2247entry:2248 br label %loop2249 2250loop:2251 %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]2252 %src.gep = getelementptr i8, ptr %src, i64 %iv2253 %load = load <20 x i8>, ptr %src.gep2254 %ext = zext <20 x i8> %load to <20 x i24>2255 %dst.gep = getelementptr i24, ptr %dst, i64 %iv2256 store <20 x i24> %ext, ptr %dst.gep2257 %iv.next = add nuw i64 %iv, 162258 %ec = icmp eq i64 %iv.next, 1282259 br i1 %ec, label %exit, label %loop2260 2261exit:2262 ret void2263}2264 2265; CHECK-LABEL: lCPI21_0:2266; CHECK-NEXT: .byte 0 ; 0x02267; CHECK-NEXT: .byte 255 ; 0xff2268; CHECK-NEXT: .byte 255 ; 0xff2269; CHECK-NEXT: .byte 255 ; 0xff2270; CHECK-NEXT: .byte 255 ; 0xff2271; CHECK-NEXT: .byte 255 ; 0xff2272; CHECK-NEXT: .byte 1 ; 0x12273; CHECK-NEXT: .byte 255 ; 0xff2274; CHECK-NEXT: .byte 255 ; 0xff2275; CHECK-NEXT: .byte 255 ; 0xff2276; CHECK-NEXT: .byte 255 ; 0xff2277; CHECK-NEXT: .byte 255 ; 0xff2278; CHECK-NEXT: .byte 2 ; 0x22279; CHECK-NEXT: .byte 255 ; 0xff2280; CHECK-NEXT: .byte 255 ; 0xff2281; CHECK-NEXT: .byte 255 ; 0xff2282; CHECK-NEXT: lCPI21_1:2283; CHECK-NEXT: .byte 255 ; 0xff2284; CHECK-NEXT: .byte 255 ; 0xff2285; CHECK-NEXT: .byte 3 ; 0x32286; CHECK-NEXT: .byte 255 ; 0xff2287; CHECK-NEXT: .byte 255 ; 0xff2288; CHECK-NEXT: .byte 255 ; 0xff2289; CHECK-NEXT: .byte 255 ; 0xff2290; CHECK-NEXT: .byte 255 ; 0xff2291; CHECK-NEXT: .byte 4 ; 0x42292; CHECK-NEXT: .byte 255 ; 0xff2293; CHECK-NEXT: .byte 255 ; 0xff2294; CHECK-NEXT: .byte 255 ; 0xff2295; CHECK-NEXT: .byte 255 ; 0xff2296; CHECK-NEXT: .byte 255 ; 0xff2297; CHECK-NEXT: .byte 5 ; 0x52298; CHECK-NEXT: .byte 255 ; 0xff2299; CHECK-NEXT: lCPI21_2:2300; CHECK-NEXT: .byte 255 ; 0xff2301; CHECK-NEXT: .byte 255 ; 0xff2302; CHECK-NEXT: .byte 255 ; 0xff2303; CHECK-NEXT: .byte 255 ; 0xff2304; CHECK-NEXT: .byte 6 ; 0x62305; CHECK-NEXT: .byte 255 ; 0xff2306; CHECK-NEXT: .byte 255 ; 0xff2307; CHECK-NEXT: .byte 255 ; 0xff2308; CHECK-NEXT: .byte 255 ; 0xff2309; CHECK-NEXT: .byte 255 ; 0xff2310; CHECK-NEXT: .byte 7 ; 0x72311; CHECK-NEXT: .byte 255 ; 0xff2312; CHECK-NEXT: .byte 255 ; 0xff2313; CHECK-NEXT: .byte 255 ; 0xff2314; CHECK-NEXT: .byte 255 ; 0xff2315; CHECK-NEXT: .byte 255 ; 0xff2316; CHECK-NEXT: lCPI21_3:2317; CHECK-NEXT: .byte 8 ; 0x82318; CHECK-NEXT: .byte 255 ; 0xff2319; CHECK-NEXT: .byte 255 ; 0xff2320; CHECK-NEXT: .byte 255 ; 0xff2321; CHECK-NEXT: .byte 255 ; 0xff2322; CHECK-NEXT: .byte 255 ; 0xff2323; CHECK-NEXT: .byte 9 ; 0x92324; CHECK-NEXT: .byte 255 ; 0xff2325; CHECK-NEXT: .byte 255 ; 0xff2326; CHECK-NEXT: .byte 255 ; 0xff2327; CHECK-NEXT: .byte 255 ; 0xff2328; CHECK-NEXT: .byte 255 ; 0xff2329; CHECK-NEXT: .byte 10 ; 0xa2330; CHECK-NEXT: .byte 255 ; 0xff2331; CHECK-NEXT: .byte 255 ; 0xff2332; CHECK-NEXT: .byte 255 ; 0xff2333; CHECK-NEXT: lCPI21_4:2334; CHECK-NEXT: .byte 255 ; 0xff2335; CHECK-NEXT: .byte 255 ; 0xff2336; CHECK-NEXT: .byte 11 ; 0xb2337; CHECK-NEXT: .byte 255 ; 0xff2338; CHECK-NEXT: .byte 255 ; 0xff2339; CHECK-NEXT: .byte 255 ; 0xff2340; CHECK-NEXT: .byte 255 ; 0xff2341; CHECK-NEXT: .byte 255 ; 0xff2342; CHECK-NEXT: .byte 12 ; 0xc2343; CHECK-NEXT: .byte 255 ; 0xff2344; CHECK-NEXT: .byte 255 ; 0xff2345; CHECK-NEXT: .byte 255 ; 0xff2346; CHECK-NEXT: .byte 255 ; 0xff2347; CHECK-NEXT: .byte 255 ; 0xff2348; CHECK-NEXT: .byte 13 ; 0xd2349; CHECK-NEXT: .byte 255 ; 0xff2350; CHECK-NEXT: lCPI21_5:2351; CHECK-NEXT: .byte 255 ; 0xff2352; CHECK-NEXT: .byte 255 ; 0xff2353; CHECK-NEXT: .byte 255 ; 0xff2354; CHECK-NEXT: .byte 255 ; 0xff2355; CHECK-NEXT: .byte 14 ; 0xe2356; CHECK-NEXT: .byte 255 ; 0xff2357; CHECK-NEXT: .byte 255 ; 0xff2358; CHECK-NEXT: .byte 255 ; 0xff2359; CHECK-NEXT: .byte 255 ; 0xff2360; CHECK-NEXT: .byte 255 ; 0xff2361; CHECK-NEXT: .byte 15 ; 0xf2362; CHECK-NEXT: .byte 255 ; 0xff2363; CHECK-NEXT: .byte 255 ; 0xff2364; CHECK-NEXT: .byte 255 ; 0xff2365; CHECK-NEXT: .byte 255 ; 0xff2366; CHECK-NEXT: .byte 255 ; 0xff2367 2368; CHECK-BE-LABEL: .LCPI21_0:2369; CHECK-BE-NEXT: .byte 255 // 0xff2370; CHECK-BE-NEXT: .byte 255 // 0xff2371; CHECK-BE-NEXT: .byte 255 // 0xff2372; CHECK-BE-NEXT: .byte 5 // 0x52373; CHECK-BE-NEXT: .byte 255 // 0xff2374; CHECK-BE-NEXT: .byte 255 // 0xff2375; CHECK-BE-NEXT: .byte 255 // 0xff2376; CHECK-BE-NEXT: .byte 255 // 0xff2377; CHECK-BE-NEXT: .byte 255 // 0xff2378; CHECK-BE-NEXT: .byte 6 // 0x62379; CHECK-BE-NEXT: .byte 255 // 0xff2380; CHECK-BE-NEXT: .byte 255 // 0xff2381; CHECK-BE-NEXT: .byte 255 // 0xff2382; CHECK-BE-NEXT: .byte 255 // 0xff2383; CHECK-BE-NEXT: .byte 255 // 0xff2384; CHECK-BE-NEXT: .byte 255 // 0xff2385; CHECK-BE-NEXT: .LCPI21_1:2386; CHECK-BE-NEXT: .byte 255 // 0xff2387; CHECK-BE-NEXT: .byte 255 // 0xff2388; CHECK-BE-NEXT: .byte 255 // 0xff2389; CHECK-BE-NEXT: .byte 255 // 0xff2390; CHECK-BE-NEXT: .byte 255 // 0xff2391; CHECK-BE-NEXT: .byte 0 // 0x02392; CHECK-BE-NEXT: .byte 255 // 0xff2393; CHECK-BE-NEXT: .byte 255 // 0xff2394; CHECK-BE-NEXT: .byte 255 // 0xff2395; CHECK-BE-NEXT: .byte 255 // 0xff2396; CHECK-BE-NEXT: .byte 255 // 0xff2397; CHECK-BE-NEXT: .byte 1 // 0x12398; CHECK-BE-NEXT: .byte 255 // 0xff2399; CHECK-BE-NEXT: .byte 255 // 0xff2400; CHECK-BE-NEXT: .byte 255 // 0xff2401; CHECK-BE-NEXT: .byte 255 // 0xff2402; CHECK-BE-NEXT: .LCPI21_2:2403; CHECK-BE-NEXT: .byte 255 // 0xff2404; CHECK-BE-NEXT: .byte 2 // 0x22405; CHECK-BE-NEXT: .byte 255 // 0xff2406; CHECK-BE-NEXT: .byte 255 // 0xff2407; CHECK-BE-NEXT: .byte 255 // 0xff2408; CHECK-BE-NEXT: .byte 255 // 0xff2409; CHECK-BE-NEXT: .byte 255 // 0xff2410; CHECK-BE-NEXT: .byte 3 // 0x32411; CHECK-BE-NEXT: .byte 255 // 0xff2412; CHECK-BE-NEXT: .byte 255 // 0xff2413; CHECK-BE-NEXT: .byte 255 // 0xff2414; CHECK-BE-NEXT: .byte 255 // 0xff2415; CHECK-BE-NEXT: .byte 255 // 0xff2416; CHECK-BE-NEXT: .byte 4 // 0x42417; CHECK-BE-NEXT: .byte 255 // 0xff2418; CHECK-BE-NEXT: .byte 255 // 0xff2419; CHECK-BE-NEXT: .LCPI21_3:2420; CHECK-BE-NEXT: .byte 255 // 0xff2421; CHECK-BE-NEXT: .byte 255 // 0xff2422; CHECK-BE-NEXT: .byte 255 // 0xff2423; CHECK-BE-NEXT: .byte 5 // 0x52424; CHECK-BE-NEXT: .byte 255 // 0xff2425; CHECK-BE-NEXT: .byte 255 // 0xff2426; CHECK-BE-NEXT: .byte 255 // 0xff2427; CHECK-BE-NEXT: .byte 255 // 0xff2428; CHECK-BE-NEXT: .byte 255 // 0xff2429; CHECK-BE-NEXT: .byte 6 // 0x62430; CHECK-BE-NEXT: .byte 255 // 0xff2431; CHECK-BE-NEXT: .byte 255 // 0xff2432; CHECK-BE-NEXT: .byte 255 // 0xff2433; CHECK-BE-NEXT: .byte 255 // 0xff2434; CHECK-BE-NEXT: .byte 255 // 0xff2435; CHECK-BE-NEXT: .byte 7 // 0x72436; CHECK-BE-NEXT: .LCPI21_4:2437; CHECK-BE-NEXT: .byte 255 // 0xff2438; CHECK-BE-NEXT: .byte 255 // 0xff2439; CHECK-BE-NEXT: .byte 255 // 0xff2440; CHECK-BE-NEXT: .byte 255 // 0xff2441; CHECK-BE-NEXT: .byte 255 // 0xff2442; CHECK-BE-NEXT: .byte 8 // 0x82443; CHECK-BE-NEXT: .byte 255 // 0xff2444; CHECK-BE-NEXT: .byte 255 // 0xff2445; CHECK-BE-NEXT: .byte 255 // 0xff2446; CHECK-BE-NEXT: .byte 255 // 0xff2447; CHECK-BE-NEXT: .byte 255 // 0xff2448; CHECK-BE-NEXT: .byte 9 // 0x92449; CHECK-BE-NEXT: .byte 255 // 0xff2450; CHECK-BE-NEXT: .byte 255 // 0xff2451; CHECK-BE-NEXT: .byte 255 // 0xff2452; CHECK-BE-NEXT: .byte 255 // 0xff2453; CHECK-BE-NEXT: .LCPI21_5:2454; CHECK-BE-NEXT: .byte 255 // 0xff2455; CHECK-BE-NEXT: .byte 10 // 0xa2456; CHECK-BE-NEXT: .byte 255 // 0xff2457; CHECK-BE-NEXT: .byte 255 // 0xff2458; CHECK-BE-NEXT: .byte 255 // 0xff2459; CHECK-BE-NEXT: .byte 255 // 0xff2460; CHECK-BE-NEXT: .byte 255 // 0xff2461; CHECK-BE-NEXT: .byte 11 // 0xb2462; CHECK-BE-NEXT: .byte 255 // 0xff2463; CHECK-BE-NEXT: .byte 255 // 0xff2464; CHECK-BE-NEXT: .byte 255 // 0xff2465; CHECK-BE-NEXT: .byte 255 // 0xff2466; CHECK-BE-NEXT: .byte 255 // 0xff2467; CHECK-BE-NEXT: .byte 12 // 0xc2468; CHECK-BE-NEXT: .byte 255 // 0xff2469; CHECK-BE-NEXT: .byte 255 // 0xff2470; CHECK-BE-NEXT: .LCPI21_6:2471; CHECK-BE-NEXT: .byte 255 // 0xff2472; CHECK-BE-NEXT: .byte 255 // 0xff2473; CHECK-BE-NEXT: .byte 255 // 0xff2474; CHECK-BE-NEXT: .byte 13 // 0xd2475; CHECK-BE-NEXT: .byte 255 // 0xff2476; CHECK-BE-NEXT: .byte 255 // 0xff2477; CHECK-BE-NEXT: .byte 255 // 0xff2478; CHECK-BE-NEXT: .byte 255 // 0xff2479; CHECK-BE-NEXT: .byte 255 // 0xff2480; CHECK-BE-NEXT: .byte 14 // 0xe2481; CHECK-BE-NEXT: .byte 255 // 0xff2482; CHECK-BE-NEXT: .byte 255 // 0xff2483; CHECK-BE-NEXT: .byte 255 // 0xff2484; CHECK-BE-NEXT: .byte 255 // 0xff2485; CHECK-BE-NEXT: .byte 255 // 0xff2486; CHECK-BE-NEXT: .byte 15 // 0xf2487 2488define void @zext_v23i8_to_v23i48_in_loop(ptr %src, ptr %dst) {2489; CHECK-LABEL: zext_v23i8_to_v23i48_in_loop:2490; CHECK: ; %bb.0: ; %entry2491; CHECK-NEXT: Lloh30:2492; CHECK-NEXT: adrp x8, lCPI21_0@PAGE2493; CHECK-NEXT: Lloh31:2494; CHECK-NEXT: adrp x9, lCPI21_1@PAGE2495; CHECK-NEXT: Lloh32:2496; CHECK-NEXT: adrp x10, lCPI21_2@PAGE2497; CHECK-NEXT: Lloh33:2498; CHECK-NEXT: ldr q0, [x8, lCPI21_0@PAGEOFF]2499; CHECK-NEXT: Lloh34:2500; CHECK-NEXT: ldr q1, [x9, lCPI21_1@PAGEOFF]2501; CHECK-NEXT: Lloh35:2502; CHECK-NEXT: ldr q2, [x10, lCPI21_2@PAGEOFF]2503; CHECK-NEXT: Lloh36:2504; CHECK-NEXT: adrp x8, lCPI21_3@PAGE2505; CHECK-NEXT: Lloh37:2506; CHECK-NEXT: adrp x9, lCPI21_4@PAGE2507; CHECK-NEXT: Lloh38:2508; CHECK-NEXT: adrp x10, lCPI21_5@PAGE2509; CHECK-NEXT: Lloh39:2510; CHECK-NEXT: ldr q3, [x8, lCPI21_3@PAGEOFF]2511; CHECK-NEXT: Lloh40:2512; CHECK-NEXT: ldr q4, [x9, lCPI21_4@PAGEOFF]2513; CHECK-NEXT: Lloh41:2514; CHECK-NEXT: ldr q5, [x10, lCPI21_5@PAGEOFF]2515; CHECK-NEXT: mov x8, xzr2516; CHECK-NEXT: LBB21_1: ; %loop2517; CHECK-NEXT: ; =>This Inner Loop Header: Depth=12518; CHECK-NEXT: add x9, x0, x82519; CHECK-NEXT: movi.2d v19, #00000000000000002520; CHECK-NEXT: add x8, x8, #162521; CHECK-NEXT: ldp q7, q6, [x9]2522; CHECK-NEXT: cmp x8, #1282523; CHECK-NEXT: strh wzr, [x1, #136]2524; CHECK-NEXT: tbl.16b v16, { v6 }, v12525; CHECK-NEXT: tbl.16b v17, { v6 }, v02526; CHECK-NEXT: mov.b v19[4], v6[6]2527; CHECK-NEXT: tbl.16b v18, { v7 }, v52528; CHECK-NEXT: tbl.16b v20, { v7 }, v42529; CHECK-NEXT: tbl.16b v21, { v7 }, v32530; CHECK-NEXT: stp q17, q16, [x1, #96]2531; CHECK-NEXT: tbl.16b v16, { v7 }, v22532; CHECK-NEXT: tbl.16b v17, { v7 }, v12533; CHECK-NEXT: tbl.16b v7, { v7 }, v02534; CHECK-NEXT: fmov x9, d192535; CHECK-NEXT: stp q20, q18, [x1, #64]2536; CHECK-NEXT: stp q16, q21, [x1, #32]2537; CHECK-NEXT: stp q7, q17, [x1]2538; CHECK-NEXT: str x9, [x1, #128]!2539; CHECK-NEXT: b.ne LBB21_12540; CHECK-NEXT: ; %bb.2: ; %exit2541; CHECK-NEXT: ret2542; CHECK-NEXT: .loh AdrpLdr Lloh38, Lloh412543; CHECK-NEXT: .loh AdrpLdr Lloh37, Lloh402544; CHECK-NEXT: .loh AdrpLdr Lloh36, Lloh392545; CHECK-NEXT: .loh AdrpAdrp Lloh32, Lloh382546; CHECK-NEXT: .loh AdrpLdr Lloh32, Lloh352547; CHECK-NEXT: .loh AdrpAdrp Lloh31, Lloh372548; CHECK-NEXT: .loh AdrpLdr Lloh31, Lloh342549; CHECK-NEXT: .loh AdrpAdrp Lloh30, Lloh362550; CHECK-NEXT: .loh AdrpLdr Lloh30, Lloh332551;2552; CHECK-BE-LABEL: zext_v23i8_to_v23i48_in_loop:2553; CHECK-BE: // %bb.0: // %entry2554; CHECK-BE-NEXT: adrp x8, .LCPI21_02555; CHECK-BE-NEXT: add x8, x8, :lo12:.LCPI21_02556; CHECK-BE-NEXT: ld1 { v0.16b }, [x8]2557; CHECK-BE-NEXT: adrp x8, .LCPI21_12558; CHECK-BE-NEXT: add x8, x8, :lo12:.LCPI21_12559; CHECK-BE-NEXT: ld1 { v1.16b }, [x8]2560; CHECK-BE-NEXT: adrp x8, .LCPI21_22561; CHECK-BE-NEXT: add x8, x8, :lo12:.LCPI21_22562; CHECK-BE-NEXT: ld1 { v2.16b }, [x8]2563; CHECK-BE-NEXT: adrp x8, .LCPI21_32564; CHECK-BE-NEXT: add x8, x8, :lo12:.LCPI21_32565; CHECK-BE-NEXT: ld1 { v3.16b }, [x8]2566; CHECK-BE-NEXT: adrp x8, .LCPI21_42567; CHECK-BE-NEXT: add x8, x8, :lo12:.LCPI21_42568; CHECK-BE-NEXT: ld1 { v4.16b }, [x8]2569; CHECK-BE-NEXT: adrp x8, .LCPI21_52570; CHECK-BE-NEXT: add x8, x8, :lo12:.LCPI21_52571; CHECK-BE-NEXT: ld1 { v5.16b }, [x8]2572; CHECK-BE-NEXT: adrp x8, .LCPI21_62573; CHECK-BE-NEXT: add x8, x8, :lo12:.LCPI21_62574; CHECK-BE-NEXT: ld1 { v6.16b }, [x8]2575; CHECK-BE-NEXT: mov x8, xzr2576; CHECK-BE-NEXT: .LBB21_1: // %loop2577; CHECK-BE-NEXT: // =>This Inner Loop Header: Depth=12578; CHECK-BE-NEXT: add x9, x0, x82579; CHECK-BE-NEXT: add x8, x8, #162580; CHECK-BE-NEXT: ld1 { v7.16b }, [x9]2581; CHECK-BE-NEXT: add x9, x9, #162582; CHECK-BE-NEXT: cmp x8, #1282583; CHECK-BE-NEXT: ld1 { v16.16b }, [x9]2584; CHECK-BE-NEXT: add x9, x1, #802585; CHECK-BE-NEXT: tbl v17.16b, { v7.16b }, v6.16b2586; CHECK-BE-NEXT: tbl v18.16b, { v7.16b }, v5.16b2587; CHECK-BE-NEXT: tbl v20.16b, { v7.16b }, v4.16b2588; CHECK-BE-NEXT: tbl v19.16b, { v16.16b }, v0.16b2589; CHECK-BE-NEXT: tbl v21.16b, { v7.16b }, v3.16b2590; CHECK-BE-NEXT: st1 { v17.16b }, [x9]2591; CHECK-BE-NEXT: add x9, x1, #642592; CHECK-BE-NEXT: tbl v17.16b, { v7.16b }, v2.16b2593; CHECK-BE-NEXT: st1 { v18.16b }, [x9]2594; CHECK-BE-NEXT: add x9, x1, #482595; CHECK-BE-NEXT: rev16 v18.16b, v19.16b2596; CHECK-BE-NEXT: st1 { v20.16b }, [x9]2597; CHECK-BE-NEXT: add x9, x1, #322598; CHECK-BE-NEXT: tbl v20.16b, { v16.16b }, v2.16b2599; CHECK-BE-NEXT: st1 { v21.16b }, [x9]2600; CHECK-BE-NEXT: add x9, x1, #162601; CHECK-BE-NEXT: tbl v16.16b, { v16.16b }, v1.16b2602; CHECK-BE-NEXT: st1 { v17.16b }, [x9]2603; CHECK-BE-NEXT: rev64 v17.16b, v19.16b2604; CHECK-BE-NEXT: add x9, x1, #1122605; CHECK-BE-NEXT: tbl v7.16b, { v7.16b }, v1.16b2606; CHECK-BE-NEXT: mov h18, v18.h[4]2607; CHECK-BE-NEXT: st1 { v20.16b }, [x9]2608; CHECK-BE-NEXT: add x9, x1, #962609; CHECK-BE-NEXT: st1 { v16.16b }, [x9]2610; CHECK-BE-NEXT: fmov x9, d172611; CHECK-BE-NEXT: st1 { v7.16b }, [x1]2612; CHECK-BE-NEXT: str h18, [x1, #136]2613; CHECK-BE-NEXT: str x9, [x1, #128]!2614; CHECK-BE-NEXT: b.ne .LBB21_12615; CHECK-BE-NEXT: // %bb.2: // %exit2616; CHECK-BE-NEXT: ret2617; CHECK : Lloh30:2618 2619 2620 2621entry:2622 br label %loop2623 2624loop:2625 %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]2626 %src.gep = getelementptr i8, ptr %src, i64 %iv2627 %load = load <23 x i8>, ptr %src.gep2628 %ext = zext <23 x i8> %load to <23 x i48>2629 %dst.gep = getelementptr i48, ptr %dst, i64 %iv2630 store <23 x i48> %ext, ptr %dst.gep2631 %iv.next = add nuw i64 %iv, 162632 %ec = icmp eq i64 %iv.next, 1282633 br i1 %ec, label %exit, label %loop2634 2635exit:2636 ret void2637}2638 2639define void @zext_v8i8_to_v8i33_in_loop(ptr %src, ptr %dst) {2640; CHECK-LABEL: zext_v8i8_to_v8i33_in_loop:2641; CHECK: ; %bb.0: ; %entry2642; CHECK-NEXT: mov x8, xzr2643; CHECK-NEXT: LBB22_1: ; %loop2644; CHECK-NEXT: ; =>This Inner Loop Header: Depth=12645; CHECK-NEXT: ldr d0, [x0, x8]2646; CHECK-NEXT: add x8, x8, #162647; CHECK-NEXT: strb wzr, [x1, #32]2648; CHECK-NEXT: cmp x8, #1282649; CHECK-NEXT: ushll.8h v0, v0, #02650; CHECK-NEXT: ushll2.4s v1, v0, #02651; CHECK-NEXT: ushll.4s v0, v0, #02652; CHECK-NEXT: ushll2.2d v2, v1, #02653; CHECK-NEXT: ushll.2d v1, v1, #02654; CHECK-NEXT: ushll2.2d v3, v0, #02655; CHECK-NEXT: ushll.2d v0, v0, #02656; CHECK-NEXT: mov.d x9, v2[1]2657; CHECK-NEXT: mov.d x10, v1[1]2658; CHECK-NEXT: fmov x12, d22659; CHECK-NEXT: mov.d x11, v3[1]2660; CHECK-NEXT: mov.d x13, v0[1]2661; CHECK-NEXT: lsl x9, x9, #392662; CHECK-NEXT: lsl x10, x10, #372663; CHECK-NEXT: lsl x11, x11, #352664; CHECK-NEXT: orr x9, x9, x12, lsl #62665; CHECK-NEXT: fmov x12, d12666; CHECK-NEXT: orr x10, x10, x12, lsl #42667; CHECK-NEXT: fmov x12, d32668; CHECK-NEXT: stp x10, x9, [x1, #16]2669; CHECK-NEXT: fmov x9, d02670; CHECK-NEXT: orr x11, x11, x12, lsl #22671; CHECK-NEXT: orr x9, x9, x13, lsl #332672; CHECK-NEXT: stp x9, x11, [x1], #1282673; CHECK-NEXT: b.ne LBB22_12674; CHECK-NEXT: ; %bb.2: ; %exit2675; CHECK-NEXT: ret2676;2677; CHECK-BE-LABEL: zext_v8i8_to_v8i33_in_loop:2678; CHECK-BE: // %bb.0: // %entry2679; CHECK-BE-NEXT: mov x8, xzr2680; CHECK-BE-NEXT: .LBB22_1: // %loop2681; CHECK-BE-NEXT: // =>This Inner Loop Header: Depth=12682; CHECK-BE-NEXT: add x9, x0, x82683; CHECK-BE-NEXT: add x8, x8, #162684; CHECK-BE-NEXT: ld1 { v0.8b }, [x9]2685; CHECK-BE-NEXT: cmp x8, #1282686; CHECK-BE-NEXT: ushll v0.8h, v0.8b, #02687; CHECK-BE-NEXT: ushll2 v1.4s, v0.8h, #02688; CHECK-BE-NEXT: ushll v0.4s, v0.4h, #02689; CHECK-BE-NEXT: ushll v2.2d, v1.2s, #02690; CHECK-BE-NEXT: ushll2 v1.2d, v1.4s, #02691; CHECK-BE-NEXT: ushll2 v3.2d, v0.4s, #02692; CHECK-BE-NEXT: ushll v0.2d, v0.2s, #02693; CHECK-BE-NEXT: mov x9, v2.d[1]2694; CHECK-BE-NEXT: mov x10, v1.d[1]2695; CHECK-BE-NEXT: fmov x13, d12696; CHECK-BE-NEXT: mov x11, v3.d[1]2697; CHECK-BE-NEXT: mov x12, v0.d[1]2698; CHECK-BE-NEXT: fmov x14, d22699; CHECK-BE-NEXT: fmov x15, d32700; CHECK-BE-NEXT: lsl x9, x9, #22701; CHECK-BE-NEXT: orr x13, x10, x13, lsl #332702; CHECK-BE-NEXT: strb w10, [x1, #32]2703; CHECK-BE-NEXT: lsl x11, x11, #42704; CHECK-BE-NEXT: lsl x12, x12, #62705; CHECK-BE-NEXT: orr x14, x9, x14, lsl #352706; CHECK-BE-NEXT: extr x9, x9, x13, #82707; CHECK-BE-NEXT: fmov x13, d02708; CHECK-BE-NEXT: orr x15, x11, x15, lsl #372709; CHECK-BE-NEXT: extr x10, x11, x14, #82710; CHECK-BE-NEXT: orr x11, x12, x13, lsl #392711; CHECK-BE-NEXT: extr x12, x12, x15, #82712; CHECK-BE-NEXT: stp x10, x9, [x1, #16]2713; CHECK-BE-NEXT: lsr x9, x11, #82714; CHECK-BE-NEXT: stp x9, x12, [x1], #1282715; CHECK-BE-NEXT: b.ne .LBB22_12716; CHECK-BE-NEXT: // %bb.2: // %exit2717; CHECK-BE-NEXT: ret2718 2719 2720entry:2721 br label %loop2722 2723loop:2724 %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]2725 %src.gep = getelementptr i8, ptr %src, i64 %iv2726 %load = load <8 x i8>, ptr %src.gep2727 %ext = zext <8 x i8> %load to <8 x i33>2728 %dst.gep = getelementptr i33, ptr %dst, i64 %iv2729 store <8 x i33> %ext, ptr %dst.gep2730 %iv.next = add nuw i64 %iv, 162731 %ec = icmp eq i64 %iv.next, 1282732 br i1 %ec, label %exit, label %loop2733 2734exit:2735 ret void2736}2737 2738; FIXME: Widening instructions should be used instead of tbl.2739define i32 @test_pr62620_widening_instr(ptr %p1, ptr %p2, i64 %lx, i32 %h) {2740; CHECK-LABEL: test_pr62620_widening_instr:2741; CHECK: ; %bb.0: ; %entry2742; CHECK-NEXT: lsl x9, x2, #42743; CHECK-NEXT: mov x8, x02744; CHECK-NEXT: mov w0, wzr2745; CHECK-NEXT: ldr q0, [x8, x9]2746; CHECK-NEXT: ldr q1, [x1, x9]2747; CHECK-NEXT: uabdl.8h v2, v0, v12748; CHECK-NEXT: uabal2.8h v2, v0, v12749; CHECK-NEXT: uaddlv.8h s0, v22750; CHECK-NEXT: fmov w8, s02751; CHECK-NEXT: LBB23_1: ; %loop2752; CHECK-NEXT: ; =>This Inner Loop Header: Depth=12753; CHECK-NEXT: subs w3, w3, #12754; CHECK-NEXT: add w0, w8, w02755; CHECK-NEXT: b.ne LBB23_12756; CHECK-NEXT: ; %bb.2: ; %exit2757; CHECK-NEXT: ret2758;2759; CHECK-BE-LABEL: test_pr62620_widening_instr:2760; CHECK-BE: // %bb.0: // %entry2761; CHECK-BE-NEXT: lsl x9, x2, #42762; CHECK-BE-NEXT: mov x8, x02763; CHECK-BE-NEXT: mov w0, wzr2764; CHECK-BE-NEXT: add x8, x8, x92765; CHECK-BE-NEXT: add x9, x1, x92766; CHECK-BE-NEXT: ld1 { v0.16b }, [x8]2767; CHECK-BE-NEXT: ld1 { v1.16b }, [x9]2768; CHECK-BE-NEXT: uabdl v2.8h, v0.8b, v1.8b2769; CHECK-BE-NEXT: uabal2 v2.8h, v0.16b, v1.16b2770; CHECK-BE-NEXT: uaddlv s0, v2.8h2771; CHECK-BE-NEXT: fmov w8, s02772; CHECK-BE-NEXT: .LBB23_1: // %loop2773; CHECK-BE-NEXT: // =>This Inner Loop Header: Depth=12774; CHECK-BE-NEXT: subs w3, w3, #12775; CHECK-BE-NEXT: add w0, w8, w02776; CHECK-BE-NEXT: b.ne .LBB23_12777; CHECK-BE-NEXT: // %bb.2: // %exit2778; CHECK-BE-NEXT: ret2779entry:2780 br label %loop2781 2782loop:2783 %s0 = phi i32 [ 0, %entry ], [ %op.rdx, %loop ]2784 %j.0261 = phi i32 [ 0, %entry ], [ %inc, %loop ]2785 %gep.1 = getelementptr inbounds <16 x i8>, ptr %p1, i64 %lx2786 %gep.2 = getelementptr inbounds <16 x i8>, ptr %p2, i64 %lx2787 %l1 = load <16 x i8>, ptr %gep.12788 %z2 = zext <16 x i8> %l1 to <16 x i32>2789 %l4 = load <16 x i8>, ptr %gep.22790 %z5 = zext <16 x i8> %l4 to <16 x i32>2791 %sub = sub nsw <16 x i32> %z2, %z52792 %abs = tail call <16 x i32> @llvm.abs.v16i32(<16 x i32> %sub, i1 true)2793 %red = tail call i32 @llvm.vector.reduce.add.v16i32(<16 x i32> %abs)2794 %op.rdx = add i32 %red, %s02795 %inc = add nuw nsw i32 %j.0261, 12796 %exitcond.not = icmp eq i32 %inc, %h2797 br i1 %exitcond.not, label %exit, label %loop2798 2799exit:2800 %s1 = phi i32 [ %op.rdx, %loop ]2801 ret i32 %s12802}2803 2804declare <16 x i32> @llvm.abs.v16i32(<16 x i32>, i1 immarg)2805 2806declare i32 @llvm.vector.reduce.add.v16i32(<16 x i32>)2807 2808define i32 @test_widening_instr_mull(ptr %p1, ptr %p2, i32 %h) {2809; CHECK-LABEL: test_widening_instr_mull:2810; CHECK: ; %bb.0: ; %entry2811; CHECK-NEXT: mov x8, x02812; CHECK-NEXT: LBB24_1: ; %loop2813; CHECK-NEXT: ; =>This Inner Loop Header: Depth=12814; CHECK-NEXT: ldr q0, [x1], #162815; CHECK-NEXT: ldr q3, [x0]2816; CHECK-NEXT: ldr q2, [x8, #16]!2817; CHECK-NEXT: subs w2, w2, #12818; CHECK-NEXT: ushll2.8h v1, v0, #02819; CHECK-NEXT: ushll.8h v0, v0, #02820; CHECK-NEXT: umull2.4s v4, v2, v12821; CHECK-NEXT: umull.4s v1, v2, v12822; CHECK-NEXT: umull2.4s v2, v3, v02823; CHECK-NEXT: umull.4s v0, v3, v02824; CHECK-NEXT: stp q1, q4, [x0, #32]2825; CHECK-NEXT: str q0, [x0]2826; CHECK-NEXT: mov x0, x82827; CHECK-NEXT: str q2, [x8]2828; CHECK-NEXT: b.ne LBB24_12829; CHECK-NEXT: ; %bb.2: ; %exit2830; CHECK-NEXT: mov w0, wzr2831; CHECK-NEXT: ret2832;2833; CHECK-BE-LABEL: test_widening_instr_mull:2834; CHECK-BE: // %bb.0: // %entry2835; CHECK-BE-NEXT: .LBB24_1: // %loop2836; CHECK-BE-NEXT: // =>This Inner Loop Header: Depth=12837; CHECK-BE-NEXT: ld1 { v0.16b }, [x1], #162838; CHECK-BE-NEXT: add x8, x0, #162839; CHECK-BE-NEXT: ld1 { v1.8h }, [x0]2840; CHECK-BE-NEXT: ld1 { v3.8h }, [x8]2841; CHECK-BE-NEXT: add x9, x0, #482842; CHECK-BE-NEXT: add x10, x0, #322843; CHECK-BE-NEXT: subs w2, w2, #12844; CHECK-BE-NEXT: ushll v2.8h, v0.8b, #02845; CHECK-BE-NEXT: ushll2 v0.8h, v0.16b, #02846; CHECK-BE-NEXT: umull v4.4s, v1.4h, v2.4h2847; CHECK-BE-NEXT: umull2 v5.4s, v3.8h, v0.8h2848; CHECK-BE-NEXT: umull v0.4s, v3.4h, v0.4h2849; CHECK-BE-NEXT: umull2 v1.4s, v1.8h, v2.8h2850; CHECK-BE-NEXT: st1 { v4.4s }, [x0]2851; CHECK-BE-NEXT: mov x0, x82852; CHECK-BE-NEXT: st1 { v5.4s }, [x9]2853; CHECK-BE-NEXT: st1 { v0.4s }, [x10]2854; CHECK-BE-NEXT: st1 { v1.4s }, [x8]2855; CHECK-BE-NEXT: b.ne .LBB24_12856; CHECK-BE-NEXT: // %bb.2: // %exit2857; CHECK-BE-NEXT: mov w0, wzr2858; CHECK-BE-NEXT: ret2859entry:2860 br label %loop2861 2862loop:2863 %iv = phi i32 [ 0, %entry ], [ %iv.next, %loop ]2864 %gep.1 = getelementptr inbounds <16 x i8>, ptr %p1, i32 %iv2865 %gep.2 = getelementptr inbounds <16 x i8>, ptr %p2, i32 %iv2866 %l1 = load <16 x i16>, ptr %gep.12867 %z2 = zext <16 x i16> %l1 to <16 x i32>2868 %l4 = load <16 x i8>, ptr %gep.22869 %z5 = zext <16 x i8> %l4 to <16 x i32>2870 %mul = mul <16 x i32> %z2, %z52871 store <16 x i32> %mul, ptr %gep.12872 %iv.next= add nuw nsw i32 %iv, 12873 %exitcond.not = icmp eq i32 %iv.next, %h2874 br i1 %exitcond.not, label %exit, label %loop2875 2876exit:2877 ret i32 02878}2879 2880define i32 @test_widening_instr_mull_64(ptr %p1, ptr %p2, i32 %h) {2881; CHECK-LABEL: test_widening_instr_mull_64:2882; CHECK: ; %bb.0: ; %entry2883; CHECK-NEXT: Lloh42:2884; CHECK-NEXT: adrp x8, lCPI25_0@PAGE2885; CHECK-NEXT: Lloh43:2886; CHECK-NEXT: adrp x9, lCPI25_1@PAGE2887; CHECK-NEXT: Lloh44:2888; CHECK-NEXT: adrp x10, lCPI25_3@PAGE2889; CHECK-NEXT: Lloh45:2890; CHECK-NEXT: ldr q0, [x8, lCPI25_0@PAGEOFF]2891; CHECK-NEXT: Lloh46:2892; CHECK-NEXT: adrp x8, lCPI25_2@PAGE2893; CHECK-NEXT: Lloh47:2894; CHECK-NEXT: ldr q1, [x9, lCPI25_1@PAGEOFF]2895; CHECK-NEXT: Lloh48:2896; CHECK-NEXT: ldr q2, [x8, lCPI25_2@PAGEOFF]2897; CHECK-NEXT: Lloh49:2898; CHECK-NEXT: ldr q3, [x10, lCPI25_3@PAGEOFF]2899; CHECK-NEXT: mov x8, x12900; CHECK-NEXT: LBB25_1: ; %loop2901; CHECK-NEXT: ; =>This Inner Loop Header: Depth=12902; CHECK-NEXT: ldr q4, [x0]2903; CHECK-NEXT: ldp q17, q16, [x1, #32]2904; CHECK-NEXT: ldr q18, [x1]2905; CHECK-NEXT: subs w2, w2, #12906; CHECK-NEXT: tbl.16b v5, { v4 }, v32907; CHECK-NEXT: tbl.16b v6, { v4 }, v02908; CHECK-NEXT: tbl.16b v7, { v4 }, v12909; CHECK-NEXT: tbl.16b v4, { v4 }, v22910; CHECK-NEXT: ldr q21, [x8, #16]!2911; CHECK-NEXT: mov x1, x82912; CHECK-NEXT: umull2.2d v19, v5, v162913; CHECK-NEXT: umull2.2d v20, v6, v172914; CHECK-NEXT: umull2.2d v22, v7, v182915; CHECK-NEXT: umull.2d v5, v5, v162916; CHECK-NEXT: umull2.2d v16, v4, v212917; CHECK-NEXT: umull.2d v4, v4, v212918; CHECK-NEXT: umull.2d v7, v7, v182919; CHECK-NEXT: umull.2d v6, v6, v172920; CHECK-NEXT: str q20, [x0, #80]2921; CHECK-NEXT: stp q22, q4, [x0, #16]2922; CHECK-NEXT: stp q5, q19, [x0, #96]2923; CHECK-NEXT: str q16, [x0, #48]2924; CHECK-NEXT: str q7, [x0]2925; CHECK-NEXT: str q6, [x0, #64]!2926; CHECK-NEXT: b.ne LBB25_12927; CHECK-NEXT: ; %bb.2: ; %exit2928; CHECK-NEXT: mov w0, wzr2929; CHECK-NEXT: ret2930; CHECK-NEXT: .loh AdrpLdr Lloh46, Lloh482931; CHECK-NEXT: .loh AdrpLdr Lloh44, Lloh492932; CHECK-NEXT: .loh AdrpLdr Lloh43, Lloh472933; CHECK-NEXT: .loh AdrpAdrp Lloh42, Lloh462934; CHECK-NEXT: .loh AdrpLdr Lloh42, Lloh452935;2936; CHECK-BE-LABEL: test_widening_instr_mull_64:2937; CHECK-BE: // %bb.0: // %entry2938; CHECK-BE-NEXT: adrp x8, .LCPI25_02939; CHECK-BE-NEXT: add x8, x8, :lo12:.LCPI25_02940; CHECK-BE-NEXT: ld1 { v0.16b }, [x8]2941; CHECK-BE-NEXT: adrp x8, .LCPI25_12942; CHECK-BE-NEXT: add x8, x8, :lo12:.LCPI25_12943; CHECK-BE-NEXT: ld1 { v1.16b }, [x8]2944; CHECK-BE-NEXT: adrp x8, .LCPI25_22945; CHECK-BE-NEXT: add x8, x8, :lo12:.LCPI25_22946; CHECK-BE-NEXT: ld1 { v2.16b }, [x8]2947; CHECK-BE-NEXT: adrp x8, .LCPI25_32948; CHECK-BE-NEXT: add x8, x8, :lo12:.LCPI25_32949; CHECK-BE-NEXT: ld1 { v3.16b }, [x8]2950; CHECK-BE-NEXT: .LBB25_1: // %loop2951; CHECK-BE-NEXT: // =>This Inner Loop Header: Depth=12952; CHECK-BE-NEXT: ld1 { v4.16b }, [x0]2953; CHECK-BE-NEXT: add x9, x1, #482954; CHECK-BE-NEXT: add x8, x1, #322955; CHECK-BE-NEXT: ld1 { v18.4s }, [x9]2956; CHECK-BE-NEXT: ld1 { v16.4s }, [x1]2957; CHECK-BE-NEXT: add x1, x1, #162958; CHECK-BE-NEXT: ld1 { v20.4s }, [x8]2959; CHECK-BE-NEXT: ld1 { v22.4s }, [x1]2960; CHECK-BE-NEXT: add x8, x0, #962961; CHECK-BE-NEXT: tbl v5.16b, { v4.16b }, v3.16b2962; CHECK-BE-NEXT: tbl v6.16b, { v4.16b }, v2.16b2963; CHECK-BE-NEXT: tbl v7.16b, { v4.16b }, v1.16b2964; CHECK-BE-NEXT: tbl v4.16b, { v4.16b }, v0.16b2965; CHECK-BE-NEXT: ext v24.16b, v18.16b, v18.16b, #82966; CHECK-BE-NEXT: add x9, x0, #322967; CHECK-BE-NEXT: ext v25.16b, v20.16b, v20.16b, #82968; CHECK-BE-NEXT: add x10, x0, #162969; CHECK-BE-NEXT: subs w2, w2, #12970; CHECK-BE-NEXT: ext v17.16b, v5.16b, v5.16b, #82971; CHECK-BE-NEXT: ext v19.16b, v6.16b, v6.16b, #82972; CHECK-BE-NEXT: rev32 v5.8b, v5.8b2973; CHECK-BE-NEXT: rev32 v21.8b, v7.8b2974; CHECK-BE-NEXT: rev32 v23.8b, v4.8b2975; CHECK-BE-NEXT: ext v7.16b, v7.16b, v7.16b, #82976; CHECK-BE-NEXT: ext v4.16b, v4.16b, v4.16b, #82977; CHECK-BE-NEXT: rev32 v6.8b, v6.8b2978; CHECK-BE-NEXT: rev32 v17.8b, v17.8b2979; CHECK-BE-NEXT: rev32 v19.8b, v19.8b2980; CHECK-BE-NEXT: umull v5.2d, v5.2s, v18.2s2981; CHECK-BE-NEXT: umull v18.2d, v21.2s, v22.2s2982; CHECK-BE-NEXT: ext v21.16b, v22.16b, v22.16b, #82983; CHECK-BE-NEXT: rev32 v7.8b, v7.8b2984; CHECK-BE-NEXT: umull v22.2d, v23.2s, v16.2s2985; CHECK-BE-NEXT: ext v16.16b, v16.16b, v16.16b, #82986; CHECK-BE-NEXT: rev32 v4.8b, v4.8b2987; CHECK-BE-NEXT: umull v17.2d, v17.2s, v24.2s2988; CHECK-BE-NEXT: umull v19.2d, v19.2s, v25.2s2989; CHECK-BE-NEXT: st1 { v5.2d }, [x8]2990; CHECK-BE-NEXT: umull v5.2d, v6.2s, v20.2s2991; CHECK-BE-NEXT: umull v6.2d, v7.2s, v21.2s2992; CHECK-BE-NEXT: add x8, x0, #1122993; CHECK-BE-NEXT: umull v4.2d, v4.2s, v16.2s2994; CHECK-BE-NEXT: st1 { v18.2d }, [x9]2995; CHECK-BE-NEXT: add x9, x0, #802996; CHECK-BE-NEXT: st1 { v22.2d }, [x0]2997; CHECK-BE-NEXT: st1 { v17.2d }, [x8]2998; CHECK-BE-NEXT: add x8, x0, #642999; CHECK-BE-NEXT: st1 { v19.2d }, [x9]3000; CHECK-BE-NEXT: add x9, x0, #483001; CHECK-BE-NEXT: mov x0, x83002; CHECK-BE-NEXT: st1 { v5.2d }, [x8]3003; CHECK-BE-NEXT: st1 { v6.2d }, [x9]3004; CHECK-BE-NEXT: st1 { v4.2d }, [x10]3005; CHECK-BE-NEXT: b.ne .LBB25_13006; CHECK-BE-NEXT: // %bb.2: // %exit3007; CHECK-BE-NEXT: mov w0, wzr3008; CHECK-BE-NEXT: ret3009entry:3010 br label %loop3011 3012loop:3013 %iv = phi i32 [ 0, %entry ], [ %iv.next, %loop ]3014 %gep.1 = getelementptr inbounds <16 x i32>, ptr %p1, i32 %iv3015 %gep.2 = getelementptr inbounds <16 x i8>, ptr %p2, i32 %iv3016 %l1 = load <16 x i8>, ptr %gep.13017 %z2 = zext <16 x i8> %l1 to <16 x i64>3018 %l4 = load <16 x i32>, ptr %gep.23019 %z5 = zext <16 x i32> %l4 to <16 x i64>3020 %mul = mul <16 x i64> %z2, %z53021 store <16 x i64> %mul, ptr %gep.13022 %iv.next= add nuw nsw i32 %iv, 13023 %exitcond.not = icmp eq i32 %iv.next, %h3024 br i1 %exitcond.not, label %exit, label %loop3025 3026exit:3027 ret i32 03028}3029 3030define i32 @test_widening_instr_mull_2(ptr %p1, ptr %p2, i32 %h) {3031; CHECK-LABEL: test_widening_instr_mull_2:3032; CHECK: ; %bb.0: ; %entry3033; CHECK-NEXT: Lloh50:3034; CHECK-NEXT: adrp x8, lCPI26_0@PAGE3035; CHECK-NEXT: Lloh51:3036; CHECK-NEXT: adrp x9, lCPI26_1@PAGE3037; CHECK-NEXT: Lloh52:3038; CHECK-NEXT: adrp x10, lCPI26_3@PAGE3039; CHECK-NEXT: Lloh53:3040; CHECK-NEXT: ldr q0, [x8, lCPI26_0@PAGEOFF]3041; CHECK-NEXT: Lloh54:3042; CHECK-NEXT: adrp x8, lCPI26_2@PAGE3043; CHECK-NEXT: Lloh55:3044; CHECK-NEXT: ldr q1, [x9, lCPI26_1@PAGEOFF]3045; CHECK-NEXT: Lloh56:3046; CHECK-NEXT: ldr q2, [x8, lCPI26_2@PAGEOFF]3047; CHECK-NEXT: Lloh57:3048; CHECK-NEXT: ldr q3, [x10, lCPI26_3@PAGEOFF]3049; CHECK-NEXT: mov x8, x03050; CHECK-NEXT: LBB26_1: ; %loop3051; CHECK-NEXT: ; =>This Inner Loop Header: Depth=13052; CHECK-NEXT: ldr q4, [x1], #163053; CHECK-NEXT: ldr q18, [x0]3054; CHECK-NEXT: ldp q16, q17, [x0, #32]3055; CHECK-NEXT: subs w2, w2, #13056; CHECK-NEXT: tbl.16b v5, { v4 }, v03057; CHECK-NEXT: tbl.16b v6, { v4 }, v13058; CHECK-NEXT: tbl.16b v7, { v4 }, v23059; CHECK-NEXT: tbl.16b v4, { v4 }, v33060; CHECK-NEXT: mul.4s v5, v16, v53061; CHECK-NEXT: ldr q16, [x8, #16]!3062; CHECK-NEXT: mul.4s v6, v17, v63063; CHECK-NEXT: mul.4s v7, v18, v73064; CHECK-NEXT: mul.4s v4, v16, v43065; CHECK-NEXT: stp q5, q6, [x0, #32]3066; CHECK-NEXT: str q7, [x0]3067; CHECK-NEXT: mov x0, x83068; CHECK-NEXT: str q4, [x8]3069; CHECK-NEXT: b.ne LBB26_13070; CHECK-NEXT: ; %bb.2: ; %exit3071; CHECK-NEXT: mov w0, wzr3072; CHECK-NEXT: ret3073; CHECK-NEXT: .loh AdrpLdr Lloh54, Lloh563074; CHECK-NEXT: .loh AdrpLdr Lloh52, Lloh573075; CHECK-NEXT: .loh AdrpLdr Lloh51, Lloh553076; CHECK-NEXT: .loh AdrpAdrp Lloh50, Lloh543077; CHECK-NEXT: .loh AdrpLdr Lloh50, Lloh533078;3079; CHECK-BE-LABEL: test_widening_instr_mull_2:3080; CHECK-BE: // %bb.0: // %entry3081; CHECK-BE-NEXT: adrp x8, .LCPI26_03082; CHECK-BE-NEXT: add x8, x8, :lo12:.LCPI26_03083; CHECK-BE-NEXT: ld1 { v0.16b }, [x8]3084; CHECK-BE-NEXT: adrp x8, .LCPI26_13085; CHECK-BE-NEXT: add x8, x8, :lo12:.LCPI26_13086; CHECK-BE-NEXT: ld1 { v1.16b }, [x8]3087; CHECK-BE-NEXT: adrp x8, .LCPI26_23088; CHECK-BE-NEXT: add x8, x8, :lo12:.LCPI26_23089; CHECK-BE-NEXT: ld1 { v2.16b }, [x8]3090; CHECK-BE-NEXT: adrp x8, .LCPI26_33091; CHECK-BE-NEXT: add x8, x8, :lo12:.LCPI26_33092; CHECK-BE-NEXT: ld1 { v3.16b }, [x8]3093; CHECK-BE-NEXT: .LBB26_1: // %loop3094; CHECK-BE-NEXT: // =>This Inner Loop Header: Depth=13095; CHECK-BE-NEXT: ld1 { v4.16b }, [x1], #163096; CHECK-BE-NEXT: add x8, x0, #323097; CHECK-BE-NEXT: ld1 { v16.4s }, [x0]3098; CHECK-BE-NEXT: add x9, x0, #483099; CHECK-BE-NEXT: add x10, x0, #163100; CHECK-BE-NEXT: ld1 { v17.4s }, [x8]3101; CHECK-BE-NEXT: ld1 { v18.4s }, [x9]3102; CHECK-BE-NEXT: ld1 { v19.4s }, [x10]3103; CHECK-BE-NEXT: subs w2, w2, #13104; CHECK-BE-NEXT: tbl v5.16b, { v4.16b }, v1.16b3105; CHECK-BE-NEXT: tbl v6.16b, { v4.16b }, v3.16b3106; CHECK-BE-NEXT: tbl v7.16b, { v4.16b }, v2.16b3107; CHECK-BE-NEXT: tbl v4.16b, { v4.16b }, v0.16b3108; CHECK-BE-NEXT: rev32 v5.16b, v5.16b3109; CHECK-BE-NEXT: rev32 v6.16b, v6.16b3110; CHECK-BE-NEXT: rev32 v7.16b, v7.16b3111; CHECK-BE-NEXT: rev32 v4.16b, v4.16b3112; CHECK-BE-NEXT: mul v5.4s, v16.4s, v5.4s3113; CHECK-BE-NEXT: mul v6.4s, v17.4s, v6.4s3114; CHECK-BE-NEXT: mul v7.4s, v18.4s, v7.4s3115; CHECK-BE-NEXT: mul v4.4s, v19.4s, v4.4s3116; CHECK-BE-NEXT: st1 { v5.4s }, [x0]3117; CHECK-BE-NEXT: mov x0, x103118; CHECK-BE-NEXT: st1 { v6.4s }, [x8]3119; CHECK-BE-NEXT: st1 { v7.4s }, [x9]3120; CHECK-BE-NEXT: st1 { v4.4s }, [x10]3121; CHECK-BE-NEXT: b.ne .LBB26_13122; CHECK-BE-NEXT: // %bb.2: // %exit3123; CHECK-BE-NEXT: mov w0, wzr3124; CHECK-BE-NEXT: ret3125entry:3126 br label %loop3127 3128loop:3129 %iv = phi i32 [ 0, %entry ], [ %iv.next, %loop ]3130 %gep.1 = getelementptr inbounds <16 x i8>, ptr %p1, i32 %iv3131 %gep.2 = getelementptr inbounds <16 x i8>, ptr %p2, i32 %iv3132 %l1 = load <16 x i32>, ptr %gep.13133 %l4 = load <16 x i8>, ptr %gep.23134 %z5 = zext <16 x i8> %l4 to <16 x i32>3135 %mul = mul <16 x i32> %l1, %z53136 store <16 x i32> %mul, ptr %gep.13137 %iv.next= add nuw nsw i32 %iv, 13138 %exitcond.not = icmp eq i32 %iv.next, %h3139 br i1 %exitcond.not, label %exit, label %loop3140 3141exit:3142 ret i32 03143}3144 3145; Not profitable to convert when there is an opportunity to partially3146; reduce the cost of widening by generating smul3147define i32 @mul_zext_16i8_sext_16i8(ptr %p1, ptr %p2, i32 %h) {3148; CHECK-LABEL: mul_zext_16i8_sext_16i8:3149; CHECK: ; %bb.0: ; %entry3150; CHECK-NEXT: LBB27_1: ; %loop3151; CHECK-NEXT: ; =>This Inner Loop Header: Depth=13152; CHECK-NEXT: ldr q0, [x0]3153; CHECK-NEXT: ldr q1, [x1], #163154; CHECK-NEXT: subs w2, w2, #13155; CHECK-NEXT: sshll2.8h v2, v0, #03156; CHECK-NEXT: ushll2.8h v3, v1, #03157; CHECK-NEXT: sshll.8h v0, v0, #03158; CHECK-NEXT: ushll.8h v1, v1, #03159; CHECK-NEXT: smull2.4s v4, v2, v33160; CHECK-NEXT: smull.4s v2, v2, v33161; CHECK-NEXT: smull.4s v3, v0, v13162; CHECK-NEXT: smull2.4s v0, v0, v13163; CHECK-NEXT: stp q2, q4, [x0, #32]3164; CHECK-NEXT: str q3, [x0]3165; CHECK-NEXT: str q0, [x0, #16]!3166; CHECK-NEXT: b.ne LBB27_13167; CHECK-NEXT: ; %bb.2: ; %exit3168; CHECK-NEXT: mov w0, wzr3169; CHECK-NEXT: ret3170;3171; CHECK-BE-LABEL: mul_zext_16i8_sext_16i8:3172; CHECK-BE: // %bb.0: // %entry3173; CHECK-BE-NEXT: .LBB27_1: // %loop3174; CHECK-BE-NEXT: // =>This Inner Loop Header: Depth=13175; CHECK-BE-NEXT: ld1 { v0.16b }, [x1], #163176; CHECK-BE-NEXT: add x8, x0, #483177; CHECK-BE-NEXT: ld1 { v1.16b }, [x0]3178; CHECK-BE-NEXT: subs w2, w2, #13179; CHECK-BE-NEXT: sshll2 v2.8h, v1.16b, #03180; CHECK-BE-NEXT: ushll2 v3.8h, v0.16b, #03181; CHECK-BE-NEXT: sshll v1.8h, v1.8b, #03182; CHECK-BE-NEXT: ushll v0.8h, v0.8b, #03183; CHECK-BE-NEXT: smull2 v4.4s, v2.8h, v3.8h3184; CHECK-BE-NEXT: smull v2.4s, v2.4h, v3.4h3185; CHECK-BE-NEXT: smull v3.4s, v1.4h, v0.4h3186; CHECK-BE-NEXT: smull2 v0.4s, v1.8h, v0.8h3187; CHECK-BE-NEXT: st1 { v4.4s }, [x8]3188; CHECK-BE-NEXT: add x8, x0, #323189; CHECK-BE-NEXT: st1 { v3.4s }, [x0]3190; CHECK-BE-NEXT: add x0, x0, #163191; CHECK-BE-NEXT: st1 { v2.4s }, [x8]3192; CHECK-BE-NEXT: st1 { v0.4s }, [x0]3193; CHECK-BE-NEXT: b.ne .LBB27_13194; CHECK-BE-NEXT: // %bb.2: // %exit3195; CHECK-BE-NEXT: mov w0, wzr3196; CHECK-BE-NEXT: ret3197entry:3198 br label %loop3199 3200loop:3201 %iv = phi i32 [ 0, %entry ], [ %iv.next, %loop ]3202 %gep.1 = getelementptr inbounds <16 x i8>, ptr %p1, i32 %iv3203 %gep.2 = getelementptr inbounds <16 x i8>, ptr %p2, i32 %iv3204 %l1 = load <16 x i8>, ptr %gep.13205 %z1 = sext <16 x i8> %l1 to <16 x i32>3206 %l4 = load <16 x i8>, ptr %gep.23207 %z5 = zext <16 x i8> %l4 to <16 x i32>3208 %mul = mul <16 x i32> %z1, %z53209 store <16 x i32> %mul, ptr %gep.13210 %iv.next= add nuw nsw i32 %iv, 13211 %exitcond.not = icmp eq i32 %iv.next, %h3212 br i1 %exitcond.not, label %exit, label %loop3213 3214exit:3215 ret i32 03216}3217 3218; Same as above but different type3219define i32 @mul_zext_16i8_sext_16i16(ptr %p1, ptr %p2, i32 %h) {3220; CHECK-LABEL: mul_zext_16i8_sext_16i16:3221; CHECK: ; %bb.0: ; %entry3222; CHECK-NEXT: mov x8, x03223; CHECK-NEXT: LBB28_1: ; %loop3224; CHECK-NEXT: ; =>This Inner Loop Header: Depth=13225; CHECK-NEXT: ldr q0, [x1], #163226; CHECK-NEXT: ldr q3, [x0]3227; CHECK-NEXT: ldr q2, [x8, #16]!3228; CHECK-NEXT: subs w2, w2, #13229; CHECK-NEXT: ushll2.8h v1, v0, #03230; CHECK-NEXT: ushll.8h v0, v0, #03231; CHECK-NEXT: smull2.4s v4, v2, v13232; CHECK-NEXT: smull.4s v1, v2, v13233; CHECK-NEXT: smull2.4s v2, v3, v03234; CHECK-NEXT: smull.4s v0, v3, v03235; CHECK-NEXT: stp q1, q4, [x0, #32]3236; CHECK-NEXT: str q0, [x0]3237; CHECK-NEXT: mov x0, x83238; CHECK-NEXT: str q2, [x8]3239; CHECK-NEXT: b.ne LBB28_13240; CHECK-NEXT: ; %bb.2: ; %exit3241; CHECK-NEXT: mov w0, wzr3242; CHECK-NEXT: ret3243;3244; CHECK-BE-LABEL: mul_zext_16i8_sext_16i16:3245; CHECK-BE: // %bb.0: // %entry3246; CHECK-BE-NEXT: .LBB28_1: // %loop3247; CHECK-BE-NEXT: // =>This Inner Loop Header: Depth=13248; CHECK-BE-NEXT: ld1 { v0.16b }, [x1], #163249; CHECK-BE-NEXT: add x8, x0, #163250; CHECK-BE-NEXT: ld1 { v1.8h }, [x0]3251; CHECK-BE-NEXT: ld1 { v3.8h }, [x8]3252; CHECK-BE-NEXT: add x9, x0, #483253; CHECK-BE-NEXT: add x10, x0, #323254; CHECK-BE-NEXT: subs w2, w2, #13255; CHECK-BE-NEXT: ushll v2.8h, v0.8b, #03256; CHECK-BE-NEXT: ushll2 v0.8h, v0.16b, #03257; CHECK-BE-NEXT: smull v4.4s, v1.4h, v2.4h3258; CHECK-BE-NEXT: smull2 v5.4s, v3.8h, v0.8h3259; CHECK-BE-NEXT: smull v0.4s, v3.4h, v0.4h3260; CHECK-BE-NEXT: smull2 v1.4s, v1.8h, v2.8h3261; CHECK-BE-NEXT: st1 { v4.4s }, [x0]3262; CHECK-BE-NEXT: mov x0, x83263; CHECK-BE-NEXT: st1 { v5.4s }, [x9]3264; CHECK-BE-NEXT: st1 { v0.4s }, [x10]3265; CHECK-BE-NEXT: st1 { v1.4s }, [x8]3266; CHECK-BE-NEXT: b.ne .LBB28_13267; CHECK-BE-NEXT: // %bb.2: // %exit3268; CHECK-BE-NEXT: mov w0, wzr3269; CHECK-BE-NEXT: ret3270entry:3271 br label %loop3272 3273loop:3274 %iv = phi i32 [ 0, %entry ], [ %iv.next, %loop ]3275 %gep.1 = getelementptr inbounds <16 x i8>, ptr %p1, i32 %iv3276 %gep.2 = getelementptr inbounds <16 x i8>, ptr %p2, i32 %iv3277 %l1 = load <16 x i16>, ptr %gep.13278 %z1 = sext <16 x i16> %l1 to <16 x i32>3279 %l4 = load <16 x i8>, ptr %gep.23280 %z5 = zext <16 x i8> %l4 to <16 x i32>3281 %mul = mul <16 x i32> %z1, %z53282 store <16 x i32> %mul, ptr %gep.13283 %iv.next= add nuw nsw i32 %iv, 13284 %exitcond.not = icmp eq i32 %iv.next, %h3285 br i1 %exitcond.not, label %exit, label %loop3286 3287exit:3288 ret i32 03289}3290