1130 lines · plain
1; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py2; RUN: llc < %s -mtriple=aarch64 -mattr=-bf16 | FileCheck %s --check-prefixes=CHECK,CHECK-CVT3; RUN: llc < %s -mtriple=aarch64 -mattr=+bf16 | FileCheck %s --check-prefixes=CHECK,CHECK-BF164 5define <8 x bfloat> @add_h(<8 x bfloat> %a, <8 x bfloat> %b) {6; CHECK-CVT-LABEL: add_h:7; CHECK-CVT: // %bb.0: // %entry8; CHECK-CVT-NEXT: shll2 v3.4s, v1.8h, #169; CHECK-CVT-NEXT: shll2 v4.4s, v0.8h, #1610; CHECK-CVT-NEXT: movi v2.4s, #111; CHECK-CVT-NEXT: ushr v5.4s, v0.4s, #1612; CHECK-CVT-NEXT: shll v1.4s, v1.4h, #1613; CHECK-CVT-NEXT: shll v0.4s, v0.4h, #1614; CHECK-CVT-NEXT: fadd v3.4s, v4.4s, v3.4s15; CHECK-CVT-NEXT: fadd v0.4s, v0.4s, v1.4s16; CHECK-CVT-NEXT: and v2.16b, v5.16b, v2.16b17; CHECK-CVT-NEXT: movi v1.4s, #127, msl #818; CHECK-CVT-NEXT: fcmeq v5.4s, v3.4s, v3.4s19; CHECK-CVT-NEXT: add v4.4s, v3.4s, v2.4s20; CHECK-CVT-NEXT: orr v3.4s, #64, lsl #1621; CHECK-CVT-NEXT: add v2.4s, v0.4s, v2.4s22; CHECK-CVT-NEXT: fcmeq v6.4s, v0.4s, v0.4s23; CHECK-CVT-NEXT: orr v0.4s, #64, lsl #1624; CHECK-CVT-NEXT: add v4.4s, v4.4s, v1.4s25; CHECK-CVT-NEXT: add v1.4s, v2.4s, v1.4s26; CHECK-CVT-NEXT: mov v2.16b, v5.16b27; CHECK-CVT-NEXT: bsl v2.16b, v4.16b, v3.16b28; CHECK-CVT-NEXT: bit v0.16b, v1.16b, v6.16b29; CHECK-CVT-NEXT: uzp2 v0.8h, v0.8h, v2.8h30; CHECK-CVT-NEXT: ret31;32; CHECK-BF16-LABEL: add_h:33; CHECK-BF16: // %bb.0: // %entry34; CHECK-BF16-NEXT: shll v2.4s, v1.4h, #1635; CHECK-BF16-NEXT: shll v3.4s, v0.4h, #1636; CHECK-BF16-NEXT: shll2 v1.4s, v1.8h, #1637; CHECK-BF16-NEXT: shll2 v0.4s, v0.8h, #1638; CHECK-BF16-NEXT: fadd v2.4s, v3.4s, v2.4s39; CHECK-BF16-NEXT: fadd v1.4s, v0.4s, v1.4s40; CHECK-BF16-NEXT: bfcvtn v0.4h, v2.4s41; CHECK-BF16-NEXT: bfcvtn2 v0.8h, v1.4s42; CHECK-BF16-NEXT: ret43entry:44 %0 = fadd <8 x bfloat> %a, %b45 ret <8 x bfloat> %046}47 48 49define <8 x bfloat> @sub_h(<8 x bfloat> %a, <8 x bfloat> %b) {50; CHECK-CVT-LABEL: sub_h:51; CHECK-CVT: // %bb.0: // %entry52; CHECK-CVT-NEXT: shll2 v3.4s, v1.8h, #1653; CHECK-CVT-NEXT: shll2 v4.4s, v0.8h, #1654; CHECK-CVT-NEXT: movi v2.4s, #155; CHECK-CVT-NEXT: ushr v5.4s, v0.4s, #1656; CHECK-CVT-NEXT: shll v1.4s, v1.4h, #1657; CHECK-CVT-NEXT: shll v0.4s, v0.4h, #1658; CHECK-CVT-NEXT: fsub v3.4s, v4.4s, v3.4s59; CHECK-CVT-NEXT: fsub v0.4s, v0.4s, v1.4s60; CHECK-CVT-NEXT: and v2.16b, v5.16b, v2.16b61; CHECK-CVT-NEXT: movi v1.4s, #127, msl #862; CHECK-CVT-NEXT: fcmeq v5.4s, v3.4s, v3.4s63; CHECK-CVT-NEXT: add v4.4s, v3.4s, v2.4s64; CHECK-CVT-NEXT: orr v3.4s, #64, lsl #1665; CHECK-CVT-NEXT: add v2.4s, v0.4s, v2.4s66; CHECK-CVT-NEXT: fcmeq v6.4s, v0.4s, v0.4s67; CHECK-CVT-NEXT: orr v0.4s, #64, lsl #1668; CHECK-CVT-NEXT: add v4.4s, v4.4s, v1.4s69; CHECK-CVT-NEXT: add v1.4s, v2.4s, v1.4s70; CHECK-CVT-NEXT: mov v2.16b, v5.16b71; CHECK-CVT-NEXT: bsl v2.16b, v4.16b, v3.16b72; CHECK-CVT-NEXT: bit v0.16b, v1.16b, v6.16b73; CHECK-CVT-NEXT: uzp2 v0.8h, v0.8h, v2.8h74; CHECK-CVT-NEXT: ret75;76; CHECK-BF16-LABEL: sub_h:77; CHECK-BF16: // %bb.0: // %entry78; CHECK-BF16-NEXT: shll v2.4s, v1.4h, #1679; CHECK-BF16-NEXT: shll v3.4s, v0.4h, #1680; CHECK-BF16-NEXT: shll2 v1.4s, v1.8h, #1681; CHECK-BF16-NEXT: shll2 v0.4s, v0.8h, #1682; CHECK-BF16-NEXT: fsub v2.4s, v3.4s, v2.4s83; CHECK-BF16-NEXT: fsub v1.4s, v0.4s, v1.4s84; CHECK-BF16-NEXT: bfcvtn v0.4h, v2.4s85; CHECK-BF16-NEXT: bfcvtn2 v0.8h, v1.4s86; CHECK-BF16-NEXT: ret87entry:88 %0 = fsub <8 x bfloat> %a, %b89 ret <8 x bfloat> %090}91 92 93define <8 x bfloat> @mul_h(<8 x bfloat> %a, <8 x bfloat> %b) {94; CHECK-CVT-LABEL: mul_h:95; CHECK-CVT: // %bb.0: // %entry96; CHECK-CVT-NEXT: shll2 v3.4s, v1.8h, #1697; CHECK-CVT-NEXT: shll2 v4.4s, v0.8h, #1698; CHECK-CVT-NEXT: movi v2.4s, #199; CHECK-CVT-NEXT: ushr v5.4s, v0.4s, #16100; CHECK-CVT-NEXT: shll v1.4s, v1.4h, #16101; CHECK-CVT-NEXT: shll v0.4s, v0.4h, #16102; CHECK-CVT-NEXT: fmul v3.4s, v4.4s, v3.4s103; CHECK-CVT-NEXT: fmul v0.4s, v0.4s, v1.4s104; CHECK-CVT-NEXT: and v2.16b, v5.16b, v2.16b105; CHECK-CVT-NEXT: movi v1.4s, #127, msl #8106; CHECK-CVT-NEXT: fcmeq v5.4s, v3.4s, v3.4s107; CHECK-CVT-NEXT: add v4.4s, v3.4s, v2.4s108; CHECK-CVT-NEXT: orr v3.4s, #64, lsl #16109; CHECK-CVT-NEXT: add v2.4s, v0.4s, v2.4s110; CHECK-CVT-NEXT: fcmeq v6.4s, v0.4s, v0.4s111; CHECK-CVT-NEXT: orr v0.4s, #64, lsl #16112; CHECK-CVT-NEXT: add v4.4s, v4.4s, v1.4s113; CHECK-CVT-NEXT: add v1.4s, v2.4s, v1.4s114; CHECK-CVT-NEXT: mov v2.16b, v5.16b115; CHECK-CVT-NEXT: bsl v2.16b, v4.16b, v3.16b116; CHECK-CVT-NEXT: bit v0.16b, v1.16b, v6.16b117; CHECK-CVT-NEXT: uzp2 v0.8h, v0.8h, v2.8h118; CHECK-CVT-NEXT: ret119;120; CHECK-BF16-LABEL: mul_h:121; CHECK-BF16: // %bb.0: // %entry122; CHECK-BF16-NEXT: shll v2.4s, v1.4h, #16123; CHECK-BF16-NEXT: shll v3.4s, v0.4h, #16124; CHECK-BF16-NEXT: shll2 v1.4s, v1.8h, #16125; CHECK-BF16-NEXT: shll2 v0.4s, v0.8h, #16126; CHECK-BF16-NEXT: fmul v2.4s, v3.4s, v2.4s127; CHECK-BF16-NEXT: fmul v1.4s, v0.4s, v1.4s128; CHECK-BF16-NEXT: bfcvtn v0.4h, v2.4s129; CHECK-BF16-NEXT: bfcvtn2 v0.8h, v1.4s130; CHECK-BF16-NEXT: ret131entry:132 %0 = fmul <8 x bfloat> %a, %b133 ret <8 x bfloat> %0134}135 136 137define <8 x bfloat> @div_h(<8 x bfloat> %a, <8 x bfloat> %b) {138; CHECK-CVT-LABEL: div_h:139; CHECK-CVT: // %bb.0: // %entry140; CHECK-CVT-NEXT: shll2 v2.4s, v1.8h, #16141; CHECK-CVT-NEXT: shll2 v3.4s, v0.8h, #16142; CHECK-CVT-NEXT: shll v1.4s, v1.4h, #16143; CHECK-CVT-NEXT: movi v4.4s, #127, msl #8144; CHECK-CVT-NEXT: fdiv v2.4s, v3.4s, v2.4s145; CHECK-CVT-NEXT: shll v3.4s, v0.4h, #16146; CHECK-CVT-NEXT: ushr v0.4s, v0.4s, #16147; CHECK-CVT-NEXT: fdiv v1.4s, v3.4s, v1.4s148; CHECK-CVT-NEXT: movi v3.4s, #1149; CHECK-CVT-NEXT: and v0.16b, v0.16b, v3.16b150; CHECK-CVT-NEXT: add v0.4s, v0.4s, v4.4s151; CHECK-CVT-NEXT: fcmeq v4.4s, v2.4s, v2.4s152; CHECK-CVT-NEXT: add v3.4s, v2.4s, v0.4s153; CHECK-CVT-NEXT: orr v2.4s, #64, lsl #16154; CHECK-CVT-NEXT: fcmeq v5.4s, v1.4s, v1.4s155; CHECK-CVT-NEXT: add v0.4s, v1.4s, v0.4s156; CHECK-CVT-NEXT: orr v1.4s, #64, lsl #16157; CHECK-CVT-NEXT: bit v2.16b, v3.16b, v4.16b158; CHECK-CVT-NEXT: bif v0.16b, v1.16b, v5.16b159; CHECK-CVT-NEXT: uzp2 v0.8h, v0.8h, v2.8h160; CHECK-CVT-NEXT: ret161;162; CHECK-BF16-LABEL: div_h:163; CHECK-BF16: // %bb.0: // %entry164; CHECK-BF16-NEXT: shll v2.4s, v1.4h, #16165; CHECK-BF16-NEXT: shll v3.4s, v0.4h, #16166; CHECK-BF16-NEXT: shll2 v1.4s, v1.8h, #16167; CHECK-BF16-NEXT: shll2 v0.4s, v0.8h, #16168; CHECK-BF16-NEXT: fdiv v2.4s, v3.4s, v2.4s169; CHECK-BF16-NEXT: fdiv v1.4s, v0.4s, v1.4s170; CHECK-BF16-NEXT: bfcvtn v0.4h, v2.4s171; CHECK-BF16-NEXT: bfcvtn2 v0.8h, v1.4s172; CHECK-BF16-NEXT: ret173entry:174 %0 = fdiv <8 x bfloat> %a, %b175 ret <8 x bfloat> %0176}177 178 179define <8 x bfloat> @load_h(ptr %a) {180; CHECK-LABEL: load_h:181; CHECK: // %bb.0: // %entry182; CHECK-NEXT: ldr q0, [x0]183; CHECK-NEXT: ret184entry:185 %0 = load <8 x bfloat>, ptr %a, align 4186 ret <8 x bfloat> %0187}188 189 190define void @store_h(ptr %a, <8 x bfloat> %b) {191; CHECK-LABEL: store_h:192; CHECK: // %bb.0: // %entry193; CHECK-NEXT: str q0, [x0]194; CHECK-NEXT: ret195entry:196 store <8 x bfloat> %b, ptr %a, align 4197 ret void198}199 200define <8 x bfloat> @s_to_h(<8 x float> %a) {201; CHECK-CVT-LABEL: s_to_h:202; CHECK-CVT: // %bb.0:203; CHECK-CVT-NEXT: movi v2.4s, #1204; CHECK-CVT-NEXT: movi v3.4s, #127, msl #8205; CHECK-CVT-NEXT: ushr v4.4s, v1.4s, #16206; CHECK-CVT-NEXT: ushr v5.4s, v0.4s, #16207; CHECK-CVT-NEXT: and v4.16b, v4.16b, v2.16b208; CHECK-CVT-NEXT: add v6.4s, v1.4s, v3.4s209; CHECK-CVT-NEXT: and v2.16b, v5.16b, v2.16b210; CHECK-CVT-NEXT: add v3.4s, v0.4s, v3.4s211; CHECK-CVT-NEXT: fcmeq v5.4s, v1.4s, v1.4s212; CHECK-CVT-NEXT: orr v1.4s, #64, lsl #16213; CHECK-CVT-NEXT: add v4.4s, v4.4s, v6.4s214; CHECK-CVT-NEXT: fcmeq v6.4s, v0.4s, v0.4s215; CHECK-CVT-NEXT: orr v0.4s, #64, lsl #16216; CHECK-CVT-NEXT: add v2.4s, v2.4s, v3.4s217; CHECK-CVT-NEXT: bit v1.16b, v4.16b, v5.16b218; CHECK-CVT-NEXT: bit v0.16b, v2.16b, v6.16b219; CHECK-CVT-NEXT: uzp2 v0.8h, v0.8h, v1.8h220; CHECK-CVT-NEXT: ret221;222; CHECK-BF16-LABEL: s_to_h:223; CHECK-BF16: // %bb.0:224; CHECK-BF16-NEXT: bfcvtn v0.4h, v0.4s225; CHECK-BF16-NEXT: bfcvtn2 v0.8h, v1.4s226; CHECK-BF16-NEXT: ret227 %1 = fptrunc <8 x float> %a to <8 x bfloat>228 ret <8 x bfloat> %1229}230 231define <8 x bfloat> @d_to_h(<8 x double> %a) {232; CHECK-CVT-LABEL: d_to_h:233; CHECK-CVT: // %bb.0:234; CHECK-CVT-NEXT: fcvtxn v2.2s, v2.2d235; CHECK-CVT-NEXT: fcvtxn v0.2s, v0.2d236; CHECK-CVT-NEXT: fcvtxn2 v2.4s, v3.2d237; CHECK-CVT-NEXT: fcvtxn2 v0.4s, v1.2d238; CHECK-CVT-NEXT: movi v1.4s, #1239; CHECK-CVT-NEXT: movi v3.4s, #127, msl #8240; CHECK-CVT-NEXT: ushr v4.4s, v2.4s, #16241; CHECK-CVT-NEXT: ushr v5.4s, v0.4s, #16242; CHECK-CVT-NEXT: add v6.4s, v2.4s, v3.4s243; CHECK-CVT-NEXT: add v3.4s, v0.4s, v3.4s244; CHECK-CVT-NEXT: and v4.16b, v4.16b, v1.16b245; CHECK-CVT-NEXT: and v1.16b, v5.16b, v1.16b246; CHECK-CVT-NEXT: fcmeq v5.4s, v2.4s, v2.4s247; CHECK-CVT-NEXT: orr v2.4s, #64, lsl #16248; CHECK-CVT-NEXT: add v4.4s, v4.4s, v6.4s249; CHECK-CVT-NEXT: fcmeq v6.4s, v0.4s, v0.4s250; CHECK-CVT-NEXT: add v1.4s, v1.4s, v3.4s251; CHECK-CVT-NEXT: orr v0.4s, #64, lsl #16252; CHECK-CVT-NEXT: bit v2.16b, v4.16b, v5.16b253; CHECK-CVT-NEXT: bit v0.16b, v1.16b, v6.16b254; CHECK-CVT-NEXT: uzp2 v0.8h, v0.8h, v2.8h255; CHECK-CVT-NEXT: ret256;257; CHECK-BF16-LABEL: d_to_h:258; CHECK-BF16: // %bb.0:259; CHECK-BF16-NEXT: fcvtxn v0.2s, v0.2d260; CHECK-BF16-NEXT: fcvtxn v2.2s, v2.2d261; CHECK-BF16-NEXT: fcvtxn2 v0.4s, v1.2d262; CHECK-BF16-NEXT: fcvtxn2 v2.4s, v3.2d263; CHECK-BF16-NEXT: bfcvtn v0.4h, v0.4s264; CHECK-BF16-NEXT: bfcvtn2 v0.8h, v2.4s265; CHECK-BF16-NEXT: ret266 %1 = fptrunc <8 x double> %a to <8 x bfloat>267 ret <8 x bfloat> %1268}269 270define <8 x float> @h_to_s(<8 x bfloat> %a) {271; CHECK-LABEL: h_to_s:272; CHECK: // %bb.0:273; CHECK-NEXT: shll2 v1.4s, v0.8h, #16274; CHECK-NEXT: shll v0.4s, v0.4h, #16275; CHECK-NEXT: ret276 %1 = fpext <8 x bfloat> %a to <8 x float>277 ret <8 x float> %1278}279 280define <8 x double> @h_to_d(<8 x bfloat> %a) {281; CHECK-LABEL: h_to_d:282; CHECK: // %bb.0:283; CHECK-NEXT: shll v1.4s, v0.4h, #16284; CHECK-NEXT: shll2 v2.4s, v0.8h, #16285; CHECK-NEXT: fcvtl v0.2d, v1.2s286; CHECK-NEXT: fcvtl2 v3.2d, v2.4s287; CHECK-NEXT: fcvtl2 v1.2d, v1.4s288; CHECK-NEXT: fcvtl v2.2d, v2.2s289; CHECK-NEXT: ret290 %1 = fpext <8 x bfloat> %a to <8 x double>291 ret <8 x double> %1292}293 294 295define <8 x bfloat> @bitcast_i_to_h(float, <8 x i16> %a) {296; CHECK-LABEL: bitcast_i_to_h:297; CHECK: // %bb.0:298; CHECK-NEXT: mov v0.16b, v1.16b299; CHECK-NEXT: ret300 %2 = bitcast <8 x i16> %a to <8 x bfloat>301 ret <8 x bfloat> %2302}303 304define <8 x i16> @bitcast_h_to_i(float, <8 x bfloat> %a) {305; CHECK-LABEL: bitcast_h_to_i:306; CHECK: // %bb.0:307; CHECK-NEXT: mov v0.16b, v1.16b308; CHECK-NEXT: ret309 %2 = bitcast <8 x bfloat> %a to <8 x i16>310 ret <8 x i16> %2311}312 313define <4 x bfloat> @sitofp_v4i8(<4 x i8> %a) #0 {314; CHECK-CVT-LABEL: sitofp_v4i8:315; CHECK-CVT: // %bb.0:316; CHECK-CVT-NEXT: shl v0.4h, v0.4h, #8317; CHECK-CVT-NEXT: movi v1.4s, #1318; CHECK-CVT-NEXT: sshr v0.4h, v0.4h, #8319; CHECK-CVT-NEXT: sshll v0.4s, v0.4h, #0320; CHECK-CVT-NEXT: scvtf v0.4s, v0.4s321; CHECK-CVT-NEXT: ushr v2.4s, v0.4s, #16322; CHECK-CVT-NEXT: and v1.16b, v2.16b, v1.16b323; CHECK-CVT-NEXT: movi v2.4s, #127, msl #8324; CHECK-CVT-NEXT: add v0.4s, v1.4s, v0.4s325; CHECK-CVT-NEXT: addhn v0.4h, v0.4s, v2.4s326; CHECK-CVT-NEXT: ret327;328; CHECK-BF16-LABEL: sitofp_v4i8:329; CHECK-BF16: // %bb.0:330; CHECK-BF16-NEXT: shl v0.4h, v0.4h, #8331; CHECK-BF16-NEXT: sshr v0.4h, v0.4h, #8332; CHECK-BF16-NEXT: sshll v0.4s, v0.4h, #0333; CHECK-BF16-NEXT: scvtf v0.4s, v0.4s334; CHECK-BF16-NEXT: bfcvtn v0.4h, v0.4s335; CHECK-BF16-NEXT: ret336 %1 = sitofp <4 x i8> %a to <4 x bfloat>337 ret <4 x bfloat> %1338}339 340define <8 x bfloat> @sitofp_v8i8(<8 x i8> %a) #0 {341; CHECK-CVT-LABEL: sitofp_v8i8:342; CHECK-CVT: // %bb.0:343; CHECK-CVT-NEXT: sshll v0.8h, v0.8b, #0344; CHECK-CVT-NEXT: movi v1.4s, #1345; CHECK-CVT-NEXT: movi v4.4s, #127, msl #8346; CHECK-CVT-NEXT: sshll v2.4s, v0.4h, #0347; CHECK-CVT-NEXT: sshll2 v0.4s, v0.8h, #0348; CHECK-CVT-NEXT: scvtf v2.4s, v2.4s349; CHECK-CVT-NEXT: scvtf v3.4s, v0.4s350; CHECK-CVT-NEXT: ushr v0.4s, v2.4s, #16351; CHECK-CVT-NEXT: ushr v5.4s, v3.4s, #16352; CHECK-CVT-NEXT: and v0.16b, v0.16b, v1.16b353; CHECK-CVT-NEXT: and v1.16b, v5.16b, v1.16b354; CHECK-CVT-NEXT: add v0.4s, v0.4s, v4.4s355; CHECK-CVT-NEXT: add v1.4s, v1.4s, v4.4s356; CHECK-CVT-NEXT: addhn v0.4h, v2.4s, v0.4s357; CHECK-CVT-NEXT: addhn2 v0.8h, v3.4s, v1.4s358; CHECK-CVT-NEXT: ret359;360; CHECK-BF16-LABEL: sitofp_v8i8:361; CHECK-BF16: // %bb.0:362; CHECK-BF16-NEXT: sshll v0.8h, v0.8b, #0363; CHECK-BF16-NEXT: sshll v1.4s, v0.4h, #0364; CHECK-BF16-NEXT: sshll2 v2.4s, v0.8h, #0365; CHECK-BF16-NEXT: scvtf v1.4s, v1.4s366; CHECK-BF16-NEXT: bfcvtn v0.4h, v1.4s367; CHECK-BF16-NEXT: scvtf v1.4s, v2.4s368; CHECK-BF16-NEXT: bfcvtn2 v0.8h, v1.4s369; CHECK-BF16-NEXT: ret370 %1 = sitofp <8 x i8> %a to <8 x bfloat>371 ret <8 x bfloat> %1372}373 374define <16 x bfloat> @sitofp_v16i8(<16 x i8> %a) #0 {375; CHECK-CVT-LABEL: sitofp_v16i8:376; CHECK-CVT: // %bb.0:377; CHECK-CVT-NEXT: sshll2 v2.8h, v0.16b, #0378; CHECK-CVT-NEXT: sshll v0.8h, v0.8b, #0379; CHECK-CVT-NEXT: movi v1.4s, #1380; CHECK-CVT-NEXT: movi v7.4s, #127, msl #8381; CHECK-CVT-NEXT: sshll v3.4s, v2.4h, #0382; CHECK-CVT-NEXT: sshll v4.4s, v0.4h, #0383; CHECK-CVT-NEXT: sshll2 v2.4s, v2.8h, #0384; CHECK-CVT-NEXT: sshll2 v0.4s, v0.8h, #0385; CHECK-CVT-NEXT: scvtf v3.4s, v3.4s386; CHECK-CVT-NEXT: scvtf v4.4s, v4.4s387; CHECK-CVT-NEXT: scvtf v2.4s, v2.4s388; CHECK-CVT-NEXT: scvtf v6.4s, v0.4s389; CHECK-CVT-NEXT: ushr v5.4s, v3.4s, #16390; CHECK-CVT-NEXT: ushr v0.4s, v4.4s, #16391; CHECK-CVT-NEXT: ushr v16.4s, v2.4s, #16392; CHECK-CVT-NEXT: ushr v17.4s, v6.4s, #16393; CHECK-CVT-NEXT: and v5.16b, v5.16b, v1.16b394; CHECK-CVT-NEXT: and v0.16b, v0.16b, v1.16b395; CHECK-CVT-NEXT: and v16.16b, v16.16b, v1.16b396; CHECK-CVT-NEXT: and v17.16b, v17.16b, v1.16b397; CHECK-CVT-NEXT: add v5.4s, v5.4s, v7.4s398; CHECK-CVT-NEXT: add v0.4s, v0.4s, v7.4s399; CHECK-CVT-NEXT: addhn v1.4h, v3.4s, v5.4s400; CHECK-CVT-NEXT: addhn v0.4h, v4.4s, v0.4s401; CHECK-CVT-NEXT: add v3.4s, v16.4s, v7.4s402; CHECK-CVT-NEXT: add v4.4s, v17.4s, v7.4s403; CHECK-CVT-NEXT: addhn2 v1.8h, v2.4s, v3.4s404; CHECK-CVT-NEXT: addhn2 v0.8h, v6.4s, v4.4s405; CHECK-CVT-NEXT: ret406;407; CHECK-BF16-LABEL: sitofp_v16i8:408; CHECK-BF16: // %bb.0:409; CHECK-BF16-NEXT: sshll2 v1.8h, v0.16b, #0410; CHECK-BF16-NEXT: sshll v0.8h, v0.8b, #0411; CHECK-BF16-NEXT: sshll v2.4s, v1.4h, #0412; CHECK-BF16-NEXT: sshll v3.4s, v0.4h, #0413; CHECK-BF16-NEXT: sshll2 v4.4s, v1.8h, #0414; CHECK-BF16-NEXT: sshll2 v5.4s, v0.8h, #0415; CHECK-BF16-NEXT: scvtf v2.4s, v2.4s416; CHECK-BF16-NEXT: scvtf v3.4s, v3.4s417; CHECK-BF16-NEXT: bfcvtn v1.4h, v2.4s418; CHECK-BF16-NEXT: scvtf v2.4s, v4.4s419; CHECK-BF16-NEXT: bfcvtn v0.4h, v3.4s420; CHECK-BF16-NEXT: scvtf v3.4s, v5.4s421; CHECK-BF16-NEXT: bfcvtn2 v1.8h, v2.4s422; CHECK-BF16-NEXT: bfcvtn2 v0.8h, v3.4s423; CHECK-BF16-NEXT: ret424 %1 = sitofp <16 x i8> %a to <16 x bfloat>425 ret <16 x bfloat> %1426}427 428define <8 x bfloat> @sitofp_i16(<8 x i16> %a) #0 {429; CHECK-CVT-LABEL: sitofp_i16:430; CHECK-CVT: // %bb.0:431; CHECK-CVT-NEXT: sshll v2.4s, v0.4h, #0432; CHECK-CVT-NEXT: sshll2 v0.4s, v0.8h, #0433; CHECK-CVT-NEXT: movi v1.4s, #1434; CHECK-CVT-NEXT: movi v4.4s, #127, msl #8435; CHECK-CVT-NEXT: scvtf v2.4s, v2.4s436; CHECK-CVT-NEXT: scvtf v3.4s, v0.4s437; CHECK-CVT-NEXT: ushr v0.4s, v2.4s, #16438; CHECK-CVT-NEXT: ushr v5.4s, v3.4s, #16439; CHECK-CVT-NEXT: and v0.16b, v0.16b, v1.16b440; CHECK-CVT-NEXT: and v1.16b, v5.16b, v1.16b441; CHECK-CVT-NEXT: add v0.4s, v0.4s, v4.4s442; CHECK-CVT-NEXT: add v1.4s, v1.4s, v4.4s443; CHECK-CVT-NEXT: addhn v0.4h, v2.4s, v0.4s444; CHECK-CVT-NEXT: addhn2 v0.8h, v3.4s, v1.4s445; CHECK-CVT-NEXT: ret446;447; CHECK-BF16-LABEL: sitofp_i16:448; CHECK-BF16: // %bb.0:449; CHECK-BF16-NEXT: sshll v1.4s, v0.4h, #0450; CHECK-BF16-NEXT: sshll2 v2.4s, v0.8h, #0451; CHECK-BF16-NEXT: scvtf v1.4s, v1.4s452; CHECK-BF16-NEXT: bfcvtn v0.4h, v1.4s453; CHECK-BF16-NEXT: scvtf v1.4s, v2.4s454; CHECK-BF16-NEXT: bfcvtn2 v0.8h, v1.4s455; CHECK-BF16-NEXT: ret456 %1 = sitofp <8 x i16> %a to <8 x bfloat>457 ret <8 x bfloat> %1458}459 460define <8 x bfloat> @sitofp_i32(<8 x i32> %a) #0 {461; CHECK-CVT-LABEL: sitofp_i32:462; CHECK-CVT: // %bb.0:463; CHECK-CVT-NEXT: scvtf v0.4s, v0.4s464; CHECK-CVT-NEXT: movi v2.4s, #1465; CHECK-CVT-NEXT: scvtf v1.4s, v1.4s466; CHECK-CVT-NEXT: movi v5.4s, #127, msl #8467; CHECK-CVT-NEXT: ushr v3.4s, v0.4s, #16468; CHECK-CVT-NEXT: ushr v4.4s, v1.4s, #16469; CHECK-CVT-NEXT: and v3.16b, v3.16b, v2.16b470; CHECK-CVT-NEXT: and v2.16b, v4.16b, v2.16b471; CHECK-CVT-NEXT: add v0.4s, v3.4s, v0.4s472; CHECK-CVT-NEXT: add v1.4s, v2.4s, v1.4s473; CHECK-CVT-NEXT: addhn v0.4h, v0.4s, v5.4s474; CHECK-CVT-NEXT: addhn2 v0.8h, v1.4s, v5.4s475; CHECK-CVT-NEXT: ret476;477; CHECK-BF16-LABEL: sitofp_i32:478; CHECK-BF16: // %bb.0:479; CHECK-BF16-NEXT: scvtf v0.4s, v0.4s480; CHECK-BF16-NEXT: scvtf v1.4s, v1.4s481; CHECK-BF16-NEXT: bfcvtn v0.4h, v0.4s482; CHECK-BF16-NEXT: bfcvtn2 v0.8h, v1.4s483; CHECK-BF16-NEXT: ret484 %1 = sitofp <8 x i32> %a to <8 x bfloat>485 ret <8 x bfloat> %1486}487 488 489define <8 x bfloat> @sitofp_i64(<8 x i64> %a) #0 {490; CHECK-CVT-LABEL: sitofp_i64:491; CHECK-CVT: // %bb.0:492; CHECK-CVT-NEXT: fmov x10, d2493; CHECK-CVT-NEXT: mov x8, v2.d[1]494; CHECK-CVT-NEXT: mov x9, v0.d[1]495; CHECK-CVT-NEXT: scvtf s2, x10496; CHECK-CVT-NEXT: fmov x10, d0497; CHECK-CVT-NEXT: scvtf s0, x8498; CHECK-CVT-NEXT: scvtf s5, x9499; CHECK-CVT-NEXT: fmov x9, d3500; CHECK-CVT-NEXT: mov x8, v3.d[1]501; CHECK-CVT-NEXT: scvtf s4, x10502; CHECK-CVT-NEXT: fmov x10, d1503; CHECK-CVT-NEXT: scvtf s3, x9504; CHECK-CVT-NEXT: mov x9, v1.d[1]505; CHECK-CVT-NEXT: mov v2.s[1], v0.s[0]506; CHECK-CVT-NEXT: scvtf s0, x10507; CHECK-CVT-NEXT: scvtf s1, x8508; CHECK-CVT-NEXT: mov v4.s[1], v5.s[0]509; CHECK-CVT-NEXT: mov v2.s[2], v3.s[0]510; CHECK-CVT-NEXT: scvtf s3, x9511; CHECK-CVT-NEXT: mov v4.s[2], v0.s[0]512; CHECK-CVT-NEXT: movi v0.4s, #1513; CHECK-CVT-NEXT: mov v2.s[3], v1.s[0]514; CHECK-CVT-NEXT: movi v1.4s, #127, msl #8515; CHECK-CVT-NEXT: mov v4.s[3], v3.s[0]516; CHECK-CVT-NEXT: ushr v3.4s, v2.4s, #16517; CHECK-CVT-NEXT: add v6.4s, v2.4s, v1.4s518; CHECK-CVT-NEXT: ushr v5.4s, v4.4s, #16519; CHECK-CVT-NEXT: add v1.4s, v4.4s, v1.4s520; CHECK-CVT-NEXT: and v3.16b, v3.16b, v0.16b521; CHECK-CVT-NEXT: and v0.16b, v5.16b, v0.16b522; CHECK-CVT-NEXT: fcmeq v5.4s, v2.4s, v2.4s523; CHECK-CVT-NEXT: orr v2.4s, #64, lsl #16524; CHECK-CVT-NEXT: add v3.4s, v3.4s, v6.4s525; CHECK-CVT-NEXT: fcmeq v6.4s, v4.4s, v4.4s526; CHECK-CVT-NEXT: orr v4.4s, #64, lsl #16527; CHECK-CVT-NEXT: add v0.4s, v0.4s, v1.4s528; CHECK-CVT-NEXT: mov v1.16b, v5.16b529; CHECK-CVT-NEXT: bif v0.16b, v4.16b, v6.16b530; CHECK-CVT-NEXT: bsl v1.16b, v3.16b, v2.16b531; CHECK-CVT-NEXT: uzp2 v0.8h, v0.8h, v1.8h532; CHECK-CVT-NEXT: ret533;534; CHECK-BF16-LABEL: sitofp_i64:535; CHECK-BF16: // %bb.0:536; CHECK-BF16-NEXT: mov x9, v0.d[1]537; CHECK-BF16-NEXT: fmov x10, d0538; CHECK-BF16-NEXT: mov x8, v2.d[1]539; CHECK-BF16-NEXT: scvtf s4, x10540; CHECK-BF16-NEXT: fmov x10, d1541; CHECK-BF16-NEXT: scvtf s0, x9542; CHECK-BF16-NEXT: fmov x9, d2543; CHECK-BF16-NEXT: scvtf s2, x8544; CHECK-BF16-NEXT: mov x8, v1.d[1]545; CHECK-BF16-NEXT: scvtf s1, x9546; CHECK-BF16-NEXT: fmov x9, d3547; CHECK-BF16-NEXT: mov v4.s[1], v0.s[0]548; CHECK-BF16-NEXT: scvtf s0, x10549; CHECK-BF16-NEXT: mov x10, v3.d[1]550; CHECK-BF16-NEXT: scvtf s3, x9551; CHECK-BF16-NEXT: mov v1.s[1], v2.s[0]552; CHECK-BF16-NEXT: scvtf s2, x8553; CHECK-BF16-NEXT: mov v4.s[2], v0.s[0]554; CHECK-BF16-NEXT: scvtf s0, x10555; CHECK-BF16-NEXT: mov v1.s[2], v3.s[0]556; CHECK-BF16-NEXT: mov v4.s[3], v2.s[0]557; CHECK-BF16-NEXT: mov v1.s[3], v0.s[0]558; CHECK-BF16-NEXT: bfcvtn v0.4h, v4.4s559; CHECK-BF16-NEXT: bfcvtn2 v0.8h, v1.4s560; CHECK-BF16-NEXT: ret561 %1 = sitofp <8 x i64> %a to <8 x bfloat>562 ret <8 x bfloat> %1563}564 565define <4 x bfloat> @uitofp_v4i8(<4 x i8> %a) #0 {566; CHECK-CVT-LABEL: uitofp_v4i8:567; CHECK-CVT: // %bb.0:568; CHECK-CVT-NEXT: bic v0.4h, #255, lsl #8569; CHECK-CVT-NEXT: movi v1.4s, #1570; CHECK-CVT-NEXT: ushll v0.4s, v0.4h, #0571; CHECK-CVT-NEXT: ucvtf v0.4s, v0.4s572; CHECK-CVT-NEXT: ushr v2.4s, v0.4s, #16573; CHECK-CVT-NEXT: and v1.16b, v2.16b, v1.16b574; CHECK-CVT-NEXT: movi v2.4s, #127, msl #8575; CHECK-CVT-NEXT: add v0.4s, v1.4s, v0.4s576; CHECK-CVT-NEXT: addhn v0.4h, v0.4s, v2.4s577; CHECK-CVT-NEXT: ret578;579; CHECK-BF16-LABEL: uitofp_v4i8:580; CHECK-BF16: // %bb.0:581; CHECK-BF16-NEXT: bic v0.4h, #255, lsl #8582; CHECK-BF16-NEXT: ushll v0.4s, v0.4h, #0583; CHECK-BF16-NEXT: ucvtf v0.4s, v0.4s584; CHECK-BF16-NEXT: bfcvtn v0.4h, v0.4s585; CHECK-BF16-NEXT: ret586 %1 = uitofp <4 x i8> %a to <4 x bfloat>587 ret <4 x bfloat> %1588}589 590define <8 x bfloat> @uitofp_v8i8(<8 x i8> %a) #0 {591; CHECK-CVT-LABEL: uitofp_v8i8:592; CHECK-CVT: // %bb.0:593; CHECK-CVT-NEXT: ushll v0.8h, v0.8b, #0594; CHECK-CVT-NEXT: movi v1.4s, #1595; CHECK-CVT-NEXT: movi v4.4s, #127, msl #8596; CHECK-CVT-NEXT: ushll v2.4s, v0.4h, #0597; CHECK-CVT-NEXT: ushll2 v0.4s, v0.8h, #0598; CHECK-CVT-NEXT: ucvtf v2.4s, v2.4s599; CHECK-CVT-NEXT: ucvtf v3.4s, v0.4s600; CHECK-CVT-NEXT: ushr v0.4s, v2.4s, #16601; CHECK-CVT-NEXT: ushr v5.4s, v3.4s, #16602; CHECK-CVT-NEXT: and v0.16b, v0.16b, v1.16b603; CHECK-CVT-NEXT: and v1.16b, v5.16b, v1.16b604; CHECK-CVT-NEXT: add v0.4s, v0.4s, v4.4s605; CHECK-CVT-NEXT: add v1.4s, v1.4s, v4.4s606; CHECK-CVT-NEXT: addhn v0.4h, v2.4s, v0.4s607; CHECK-CVT-NEXT: addhn2 v0.8h, v3.4s, v1.4s608; CHECK-CVT-NEXT: ret609;610; CHECK-BF16-LABEL: uitofp_v8i8:611; CHECK-BF16: // %bb.0:612; CHECK-BF16-NEXT: ushll v0.8h, v0.8b, #0613; CHECK-BF16-NEXT: ushll v1.4s, v0.4h, #0614; CHECK-BF16-NEXT: ushll2 v2.4s, v0.8h, #0615; CHECK-BF16-NEXT: ucvtf v1.4s, v1.4s616; CHECK-BF16-NEXT: bfcvtn v0.4h, v1.4s617; CHECK-BF16-NEXT: ucvtf v1.4s, v2.4s618; CHECK-BF16-NEXT: bfcvtn2 v0.8h, v1.4s619; CHECK-BF16-NEXT: ret620 %1 = uitofp <8 x i8> %a to <8 x bfloat>621 ret <8 x bfloat> %1622}623 624define <16 x bfloat> @uitofp_v16i8(<16 x i8> %a) #0 {625; CHECK-CVT-LABEL: uitofp_v16i8:626; CHECK-CVT: // %bb.0:627; CHECK-CVT-NEXT: ushll2 v2.8h, v0.16b, #0628; CHECK-CVT-NEXT: ushll v0.8h, v0.8b, #0629; CHECK-CVT-NEXT: movi v1.4s, #1630; CHECK-CVT-NEXT: movi v7.4s, #127, msl #8631; CHECK-CVT-NEXT: ushll v3.4s, v2.4h, #0632; CHECK-CVT-NEXT: ushll v4.4s, v0.4h, #0633; CHECK-CVT-NEXT: ushll2 v2.4s, v2.8h, #0634; CHECK-CVT-NEXT: ushll2 v0.4s, v0.8h, #0635; CHECK-CVT-NEXT: ucvtf v3.4s, v3.4s636; CHECK-CVT-NEXT: ucvtf v4.4s, v4.4s637; CHECK-CVT-NEXT: ucvtf v2.4s, v2.4s638; CHECK-CVT-NEXT: ucvtf v6.4s, v0.4s639; CHECK-CVT-NEXT: ushr v5.4s, v3.4s, #16640; CHECK-CVT-NEXT: ushr v0.4s, v4.4s, #16641; CHECK-CVT-NEXT: ushr v16.4s, v2.4s, #16642; CHECK-CVT-NEXT: ushr v17.4s, v6.4s, #16643; CHECK-CVT-NEXT: and v5.16b, v5.16b, v1.16b644; CHECK-CVT-NEXT: and v0.16b, v0.16b, v1.16b645; CHECK-CVT-NEXT: and v16.16b, v16.16b, v1.16b646; CHECK-CVT-NEXT: and v17.16b, v17.16b, v1.16b647; CHECK-CVT-NEXT: add v5.4s, v5.4s, v7.4s648; CHECK-CVT-NEXT: add v0.4s, v0.4s, v7.4s649; CHECK-CVT-NEXT: addhn v1.4h, v3.4s, v5.4s650; CHECK-CVT-NEXT: addhn v0.4h, v4.4s, v0.4s651; CHECK-CVT-NEXT: add v3.4s, v16.4s, v7.4s652; CHECK-CVT-NEXT: add v4.4s, v17.4s, v7.4s653; CHECK-CVT-NEXT: addhn2 v1.8h, v2.4s, v3.4s654; CHECK-CVT-NEXT: addhn2 v0.8h, v6.4s, v4.4s655; CHECK-CVT-NEXT: ret656;657; CHECK-BF16-LABEL: uitofp_v16i8:658; CHECK-BF16: // %bb.0:659; CHECK-BF16-NEXT: ushll2 v1.8h, v0.16b, #0660; CHECK-BF16-NEXT: ushll v0.8h, v0.8b, #0661; CHECK-BF16-NEXT: ushll v2.4s, v1.4h, #0662; CHECK-BF16-NEXT: ushll v3.4s, v0.4h, #0663; CHECK-BF16-NEXT: ushll2 v4.4s, v1.8h, #0664; CHECK-BF16-NEXT: ushll2 v5.4s, v0.8h, #0665; CHECK-BF16-NEXT: ucvtf v2.4s, v2.4s666; CHECK-BF16-NEXT: ucvtf v3.4s, v3.4s667; CHECK-BF16-NEXT: bfcvtn v1.4h, v2.4s668; CHECK-BF16-NEXT: ucvtf v2.4s, v4.4s669; CHECK-BF16-NEXT: bfcvtn v0.4h, v3.4s670; CHECK-BF16-NEXT: ucvtf v3.4s, v5.4s671; CHECK-BF16-NEXT: bfcvtn2 v1.8h, v2.4s672; CHECK-BF16-NEXT: bfcvtn2 v0.8h, v3.4s673; CHECK-BF16-NEXT: ret674 %1 = uitofp <16 x i8> %a to <16 x bfloat>675 ret <16 x bfloat> %1676}677 678 679define <8 x bfloat> @uitofp_i16(<8 x i16> %a) #0 {680; CHECK-CVT-LABEL: uitofp_i16:681; CHECK-CVT: // %bb.0:682; CHECK-CVT-NEXT: ushll v2.4s, v0.4h, #0683; CHECK-CVT-NEXT: ushll2 v0.4s, v0.8h, #0684; CHECK-CVT-NEXT: movi v1.4s, #1685; CHECK-CVT-NEXT: movi v4.4s, #127, msl #8686; CHECK-CVT-NEXT: ucvtf v2.4s, v2.4s687; CHECK-CVT-NEXT: ucvtf v3.4s, v0.4s688; CHECK-CVT-NEXT: ushr v0.4s, v2.4s, #16689; CHECK-CVT-NEXT: ushr v5.4s, v3.4s, #16690; CHECK-CVT-NEXT: and v0.16b, v0.16b, v1.16b691; CHECK-CVT-NEXT: and v1.16b, v5.16b, v1.16b692; CHECK-CVT-NEXT: add v0.4s, v0.4s, v4.4s693; CHECK-CVT-NEXT: add v1.4s, v1.4s, v4.4s694; CHECK-CVT-NEXT: addhn v0.4h, v2.4s, v0.4s695; CHECK-CVT-NEXT: addhn2 v0.8h, v3.4s, v1.4s696; CHECK-CVT-NEXT: ret697;698; CHECK-BF16-LABEL: uitofp_i16:699; CHECK-BF16: // %bb.0:700; CHECK-BF16-NEXT: ushll v1.4s, v0.4h, #0701; CHECK-BF16-NEXT: ushll2 v2.4s, v0.8h, #0702; CHECK-BF16-NEXT: ucvtf v1.4s, v1.4s703; CHECK-BF16-NEXT: bfcvtn v0.4h, v1.4s704; CHECK-BF16-NEXT: ucvtf v1.4s, v2.4s705; CHECK-BF16-NEXT: bfcvtn2 v0.8h, v1.4s706; CHECK-BF16-NEXT: ret707 %1 = uitofp <8 x i16> %a to <8 x bfloat>708 ret <8 x bfloat> %1709}710 711 712define <8 x bfloat> @uitofp_i32(<8 x i32> %a) #0 {713; CHECK-CVT-LABEL: uitofp_i32:714; CHECK-CVT: // %bb.0:715; CHECK-CVT-NEXT: ucvtf v0.4s, v0.4s716; CHECK-CVT-NEXT: movi v2.4s, #1717; CHECK-CVT-NEXT: ucvtf v1.4s, v1.4s718; CHECK-CVT-NEXT: movi v5.4s, #127, msl #8719; CHECK-CVT-NEXT: ushr v3.4s, v0.4s, #16720; CHECK-CVT-NEXT: ushr v4.4s, v1.4s, #16721; CHECK-CVT-NEXT: and v3.16b, v3.16b, v2.16b722; CHECK-CVT-NEXT: and v2.16b, v4.16b, v2.16b723; CHECK-CVT-NEXT: add v0.4s, v3.4s, v0.4s724; CHECK-CVT-NEXT: add v1.4s, v2.4s, v1.4s725; CHECK-CVT-NEXT: addhn v0.4h, v0.4s, v5.4s726; CHECK-CVT-NEXT: addhn2 v0.8h, v1.4s, v5.4s727; CHECK-CVT-NEXT: ret728;729; CHECK-BF16-LABEL: uitofp_i32:730; CHECK-BF16: // %bb.0:731; CHECK-BF16-NEXT: ucvtf v0.4s, v0.4s732; CHECK-BF16-NEXT: ucvtf v1.4s, v1.4s733; CHECK-BF16-NEXT: bfcvtn v0.4h, v0.4s734; CHECK-BF16-NEXT: bfcvtn2 v0.8h, v1.4s735; CHECK-BF16-NEXT: ret736 %1 = uitofp <8 x i32> %a to <8 x bfloat>737 ret <8 x bfloat> %1738}739 740 741define <8 x bfloat> @uitofp_i64(<8 x i64> %a) #0 {742; CHECK-CVT-LABEL: uitofp_i64:743; CHECK-CVT: // %bb.0:744; CHECK-CVT-NEXT: fmov x10, d2745; CHECK-CVT-NEXT: mov x8, v2.d[1]746; CHECK-CVT-NEXT: mov x9, v0.d[1]747; CHECK-CVT-NEXT: ucvtf s2, x10748; CHECK-CVT-NEXT: fmov x10, d0749; CHECK-CVT-NEXT: ucvtf s0, x8750; CHECK-CVT-NEXT: ucvtf s5, x9751; CHECK-CVT-NEXT: fmov x9, d3752; CHECK-CVT-NEXT: mov x8, v3.d[1]753; CHECK-CVT-NEXT: ucvtf s4, x10754; CHECK-CVT-NEXT: fmov x10, d1755; CHECK-CVT-NEXT: ucvtf s3, x9756; CHECK-CVT-NEXT: mov x9, v1.d[1]757; CHECK-CVT-NEXT: mov v2.s[1], v0.s[0]758; CHECK-CVT-NEXT: ucvtf s0, x10759; CHECK-CVT-NEXT: ucvtf s1, x8760; CHECK-CVT-NEXT: mov v4.s[1], v5.s[0]761; CHECK-CVT-NEXT: mov v2.s[2], v3.s[0]762; CHECK-CVT-NEXT: ucvtf s3, x9763; CHECK-CVT-NEXT: mov v4.s[2], v0.s[0]764; CHECK-CVT-NEXT: movi v0.4s, #1765; CHECK-CVT-NEXT: mov v2.s[3], v1.s[0]766; CHECK-CVT-NEXT: movi v1.4s, #127, msl #8767; CHECK-CVT-NEXT: mov v4.s[3], v3.s[0]768; CHECK-CVT-NEXT: ushr v3.4s, v2.4s, #16769; CHECK-CVT-NEXT: add v6.4s, v2.4s, v1.4s770; CHECK-CVT-NEXT: ushr v5.4s, v4.4s, #16771; CHECK-CVT-NEXT: add v1.4s, v4.4s, v1.4s772; CHECK-CVT-NEXT: and v3.16b, v3.16b, v0.16b773; CHECK-CVT-NEXT: and v0.16b, v5.16b, v0.16b774; CHECK-CVT-NEXT: fcmeq v5.4s, v2.4s, v2.4s775; CHECK-CVT-NEXT: orr v2.4s, #64, lsl #16776; CHECK-CVT-NEXT: add v3.4s, v3.4s, v6.4s777; CHECK-CVT-NEXT: fcmeq v6.4s, v4.4s, v4.4s778; CHECK-CVT-NEXT: orr v4.4s, #64, lsl #16779; CHECK-CVT-NEXT: add v0.4s, v0.4s, v1.4s780; CHECK-CVT-NEXT: mov v1.16b, v5.16b781; CHECK-CVT-NEXT: bif v0.16b, v4.16b, v6.16b782; CHECK-CVT-NEXT: bsl v1.16b, v3.16b, v2.16b783; CHECK-CVT-NEXT: uzp2 v0.8h, v0.8h, v1.8h784; CHECK-CVT-NEXT: ret785;786; CHECK-BF16-LABEL: uitofp_i64:787; CHECK-BF16: // %bb.0:788; CHECK-BF16-NEXT: mov x9, v0.d[1]789; CHECK-BF16-NEXT: fmov x10, d0790; CHECK-BF16-NEXT: mov x8, v2.d[1]791; CHECK-BF16-NEXT: ucvtf s4, x10792; CHECK-BF16-NEXT: fmov x10, d1793; CHECK-BF16-NEXT: ucvtf s0, x9794; CHECK-BF16-NEXT: fmov x9, d2795; CHECK-BF16-NEXT: ucvtf s2, x8796; CHECK-BF16-NEXT: mov x8, v1.d[1]797; CHECK-BF16-NEXT: ucvtf s1, x9798; CHECK-BF16-NEXT: fmov x9, d3799; CHECK-BF16-NEXT: mov v4.s[1], v0.s[0]800; CHECK-BF16-NEXT: ucvtf s0, x10801; CHECK-BF16-NEXT: mov x10, v3.d[1]802; CHECK-BF16-NEXT: ucvtf s3, x9803; CHECK-BF16-NEXT: mov v1.s[1], v2.s[0]804; CHECK-BF16-NEXT: ucvtf s2, x8805; CHECK-BF16-NEXT: mov v4.s[2], v0.s[0]806; CHECK-BF16-NEXT: ucvtf s0, x10807; CHECK-BF16-NEXT: mov v1.s[2], v3.s[0]808; CHECK-BF16-NEXT: mov v4.s[3], v2.s[0]809; CHECK-BF16-NEXT: mov v1.s[3], v0.s[0]810; CHECK-BF16-NEXT: bfcvtn v0.4h, v4.4s811; CHECK-BF16-NEXT: bfcvtn2 v0.8h, v1.4s812; CHECK-BF16-NEXT: ret813 %1 = uitofp <8 x i64> %a to <8 x bfloat>814 ret <8 x bfloat> %1815}816 817define void @test_insert_at_zero(bfloat %a, ptr %b) #0 {818; CHECK-LABEL: test_insert_at_zero:819; CHECK: // %bb.0:820; CHECK-NEXT: // kill: def $h0 killed $h0 def $q0821; CHECK-NEXT: str q0, [x0]822; CHECK-NEXT: ret823 %1 = insertelement <8 x bfloat> undef, bfloat %a, i64 0824 store <8 x bfloat> %1, ptr %b, align 4825 ret void826}827 828define <8 x i8> @fptosi_i8(<8 x bfloat> %a) #0 {829; CHECK-LABEL: fptosi_i8:830; CHECK: // %bb.0:831; CHECK-NEXT: shll2 v1.4s, v0.8h, #16832; CHECK-NEXT: shll v0.4s, v0.4h, #16833; CHECK-NEXT: fcvtzs v1.4s, v1.4s834; CHECK-NEXT: fcvtzs v0.4s, v0.4s835; CHECK-NEXT: uzp1 v0.8h, v0.8h, v1.8h836; CHECK-NEXT: xtn v0.8b, v0.8h837; CHECK-NEXT: ret838 %1 = fptosi<8 x bfloat> %a to <8 x i8>839 ret <8 x i8> %1840}841 842define <8 x i16> @fptosi_i16(<8 x bfloat> %a) #0 {843; CHECK-LABEL: fptosi_i16:844; CHECK: // %bb.0:845; CHECK-NEXT: shll2 v1.4s, v0.8h, #16846; CHECK-NEXT: shll v0.4s, v0.4h, #16847; CHECK-NEXT: fcvtzs v1.4s, v1.4s848; CHECK-NEXT: fcvtzs v0.4s, v0.4s849; CHECK-NEXT: uzp1 v0.8h, v0.8h, v1.8h850; CHECK-NEXT: ret851 %1 = fptosi<8 x bfloat> %a to <8 x i16>852 ret <8 x i16> %1853}854 855define <8 x i8> @fptoui_i8(<8 x bfloat> %a) #0 {856; CHECK-LABEL: fptoui_i8:857; CHECK: // %bb.0:858; CHECK-NEXT: shll2 v1.4s, v0.8h, #16859; CHECK-NEXT: shll v0.4s, v0.4h, #16860; CHECK-NEXT: fcvtzu v1.4s, v1.4s861; CHECK-NEXT: fcvtzu v0.4s, v0.4s862; CHECK-NEXT: uzp1 v0.8h, v0.8h, v1.8h863; CHECK-NEXT: xtn v0.8b, v0.8h864; CHECK-NEXT: ret865 %1 = fptoui<8 x bfloat> %a to <8 x i8>866 ret <8 x i8> %1867}868 869define <8 x i16> @fptoui_i16(<8 x bfloat> %a) #0 {870; CHECK-LABEL: fptoui_i16:871; CHECK: // %bb.0:872; CHECK-NEXT: shll2 v1.4s, v0.8h, #16873; CHECK-NEXT: shll v0.4s, v0.4h, #16874; CHECK-NEXT: fcvtzu v1.4s, v1.4s875; CHECK-NEXT: fcvtzu v0.4s, v0.4s876; CHECK-NEXT: uzp1 v0.8h, v0.8h, v1.8h877; CHECK-NEXT: ret878 %1 = fptoui<8 x bfloat> %a to <8 x i16>879 ret <8 x i16> %1880}881 882define <8 x i1> @test_fcmp_une(<8 x bfloat> %a, <8 x bfloat> %b) #0 {883; CHECK-LABEL: test_fcmp_une:884; CHECK: // %bb.0:885; CHECK-NEXT: shll2 v2.4s, v1.8h, #16886; CHECK-NEXT: shll2 v3.4s, v0.8h, #16887; CHECK-NEXT: shll v1.4s, v1.4h, #16888; CHECK-NEXT: shll v0.4s, v0.4h, #16889; CHECK-NEXT: fcmeq v2.4s, v3.4s, v2.4s890; CHECK-NEXT: fcmeq v0.4s, v0.4s, v1.4s891; CHECK-NEXT: uzp1 v0.8h, v0.8h, v2.8h892; CHECK-NEXT: mvn v0.16b, v0.16b893; CHECK-NEXT: xtn v0.8b, v0.8h894; CHECK-NEXT: ret895 %1 = fcmp une <8 x bfloat> %a, %b896 ret <8 x i1> %1897}898 899define <8 x i1> @test_fcmp_ueq(<8 x bfloat> %a, <8 x bfloat> %b) #0 {900; CHECK-LABEL: test_fcmp_ueq:901; CHECK: // %bb.0:902; CHECK-NEXT: shll2 v2.4s, v1.8h, #16903; CHECK-NEXT: shll2 v3.4s, v0.8h, #16904; CHECK-NEXT: shll v1.4s, v1.4h, #16905; CHECK-NEXT: shll v0.4s, v0.4h, #16906; CHECK-NEXT: fcmgt v4.4s, v3.4s, v2.4s907; CHECK-NEXT: fcmgt v2.4s, v2.4s, v3.4s908; CHECK-NEXT: fcmgt v3.4s, v0.4s, v1.4s909; CHECK-NEXT: fcmgt v0.4s, v1.4s, v0.4s910; CHECK-NEXT: orr v1.16b, v2.16b, v4.16b911; CHECK-NEXT: orr v0.16b, v0.16b, v3.16b912; CHECK-NEXT: uzp1 v0.8h, v0.8h, v1.8h913; CHECK-NEXT: mvn v0.16b, v0.16b914; CHECK-NEXT: xtn v0.8b, v0.8h915; CHECK-NEXT: ret916 %1 = fcmp ueq <8 x bfloat> %a, %b917 ret <8 x i1> %1918}919 920define <8 x i1> @test_fcmp_ugt(<8 x bfloat> %a, <8 x bfloat> %b) #0 {921; CHECK-LABEL: test_fcmp_ugt:922; CHECK: // %bb.0:923; CHECK-NEXT: shll2 v2.4s, v0.8h, #16924; CHECK-NEXT: shll2 v3.4s, v1.8h, #16925; CHECK-NEXT: shll v0.4s, v0.4h, #16926; CHECK-NEXT: shll v1.4s, v1.4h, #16927; CHECK-NEXT: fcmge v2.4s, v3.4s, v2.4s928; CHECK-NEXT: fcmge v0.4s, v1.4s, v0.4s929; CHECK-NEXT: uzp1 v0.8h, v0.8h, v2.8h930; CHECK-NEXT: mvn v0.16b, v0.16b931; CHECK-NEXT: xtn v0.8b, v0.8h932; CHECK-NEXT: ret933 %1 = fcmp ugt <8 x bfloat> %a, %b934 ret <8 x i1> %1935}936 937define <8 x i1> @test_fcmp_uge(<8 x bfloat> %a, <8 x bfloat> %b) #0 {938; CHECK-LABEL: test_fcmp_uge:939; CHECK: // %bb.0:940; CHECK-NEXT: shll2 v2.4s, v0.8h, #16941; CHECK-NEXT: shll2 v3.4s, v1.8h, #16942; CHECK-NEXT: shll v0.4s, v0.4h, #16943; CHECK-NEXT: shll v1.4s, v1.4h, #16944; CHECK-NEXT: fcmgt v2.4s, v3.4s, v2.4s945; CHECK-NEXT: fcmgt v0.4s, v1.4s, v0.4s946; CHECK-NEXT: uzp1 v0.8h, v0.8h, v2.8h947; CHECK-NEXT: mvn v0.16b, v0.16b948; CHECK-NEXT: xtn v0.8b, v0.8h949; CHECK-NEXT: ret950 %1 = fcmp uge <8 x bfloat> %a, %b951 ret <8 x i1> %1952}953 954define <8 x i1> @test_fcmp_ult(<8 x bfloat> %a, <8 x bfloat> %b) #0 {955; CHECK-LABEL: test_fcmp_ult:956; CHECK: // %bb.0:957; CHECK-NEXT: shll2 v2.4s, v1.8h, #16958; CHECK-NEXT: shll2 v3.4s, v0.8h, #16959; CHECK-NEXT: shll v1.4s, v1.4h, #16960; CHECK-NEXT: shll v0.4s, v0.4h, #16961; CHECK-NEXT: fcmge v2.4s, v3.4s, v2.4s962; CHECK-NEXT: fcmge v0.4s, v0.4s, v1.4s963; CHECK-NEXT: uzp1 v0.8h, v0.8h, v2.8h964; CHECK-NEXT: mvn v0.16b, v0.16b965; CHECK-NEXT: xtn v0.8b, v0.8h966; CHECK-NEXT: ret967 %1 = fcmp ult <8 x bfloat> %a, %b968 ret <8 x i1> %1969}970 971define <8 x i1> @test_fcmp_ule(<8 x bfloat> %a, <8 x bfloat> %b) #0 {972; CHECK-LABEL: test_fcmp_ule:973; CHECK: // %bb.0:974; CHECK-NEXT: shll2 v2.4s, v1.8h, #16975; CHECK-NEXT: shll2 v3.4s, v0.8h, #16976; CHECK-NEXT: shll v1.4s, v1.4h, #16977; CHECK-NEXT: shll v0.4s, v0.4h, #16978; CHECK-NEXT: fcmgt v2.4s, v3.4s, v2.4s979; CHECK-NEXT: fcmgt v0.4s, v0.4s, v1.4s980; CHECK-NEXT: uzp1 v0.8h, v0.8h, v2.8h981; CHECK-NEXT: mvn v0.16b, v0.16b982; CHECK-NEXT: xtn v0.8b, v0.8h983; CHECK-NEXT: ret984 %1 = fcmp ule <8 x bfloat> %a, %b985 ret <8 x i1> %1986}987 988define <8 x i1> @test_fcmp_uno(<8 x bfloat> %a, <8 x bfloat> %b) #0 {989; CHECK-LABEL: test_fcmp_uno:990; CHECK: // %bb.0:991; CHECK-NEXT: shll2 v2.4s, v1.8h, #16992; CHECK-NEXT: shll2 v3.4s, v0.8h, #16993; CHECK-NEXT: shll v1.4s, v1.4h, #16994; CHECK-NEXT: shll v0.4s, v0.4h, #16995; CHECK-NEXT: fcmge v4.4s, v3.4s, v2.4s996; CHECK-NEXT: fcmgt v2.4s, v2.4s, v3.4s997; CHECK-NEXT: fcmge v3.4s, v0.4s, v1.4s998; CHECK-NEXT: fcmgt v0.4s, v1.4s, v0.4s999; CHECK-NEXT: orr v1.16b, v2.16b, v4.16b1000; CHECK-NEXT: orr v0.16b, v0.16b, v3.16b1001; CHECK-NEXT: uzp1 v0.8h, v0.8h, v1.8h1002; CHECK-NEXT: mvn v0.16b, v0.16b1003; CHECK-NEXT: xtn v0.8b, v0.8h1004; CHECK-NEXT: ret1005 %1 = fcmp uno <8 x bfloat> %a, %b1006 ret <8 x i1> %11007}1008 1009define <8 x i1> @test_fcmp_one(<8 x bfloat> %a, <8 x bfloat> %b) #0 {1010; CHECK-LABEL: test_fcmp_one:1011; CHECK: // %bb.0:1012; CHECK-NEXT: shll2 v2.4s, v1.8h, #161013; CHECK-NEXT: shll2 v3.4s, v0.8h, #161014; CHECK-NEXT: shll v1.4s, v1.4h, #161015; CHECK-NEXT: shll v0.4s, v0.4h, #161016; CHECK-NEXT: fcmgt v4.4s, v3.4s, v2.4s1017; CHECK-NEXT: fcmgt v2.4s, v2.4s, v3.4s1018; CHECK-NEXT: fcmgt v3.4s, v0.4s, v1.4s1019; CHECK-NEXT: fcmgt v0.4s, v1.4s, v0.4s1020; CHECK-NEXT: orr v1.16b, v2.16b, v4.16b1021; CHECK-NEXT: orr v0.16b, v0.16b, v3.16b1022; CHECK-NEXT: uzp1 v0.8h, v0.8h, v1.8h1023; CHECK-NEXT: xtn v0.8b, v0.8h1024; CHECK-NEXT: ret1025 %1 = fcmp one <8 x bfloat> %a, %b1026 ret <8 x i1> %11027}1028 1029define <8 x i1> @test_fcmp_oeq(<8 x bfloat> %a, <8 x bfloat> %b) #0 {1030; CHECK-LABEL: test_fcmp_oeq:1031; CHECK: // %bb.0:1032; CHECK-NEXT: shll2 v2.4s, v1.8h, #161033; CHECK-NEXT: shll2 v3.4s, v0.8h, #161034; CHECK-NEXT: shll v1.4s, v1.4h, #161035; CHECK-NEXT: shll v0.4s, v0.4h, #161036; CHECK-NEXT: fcmeq v2.4s, v3.4s, v2.4s1037; CHECK-NEXT: fcmeq v0.4s, v0.4s, v1.4s1038; CHECK-NEXT: uzp1 v0.8h, v0.8h, v2.8h1039; CHECK-NEXT: xtn v0.8b, v0.8h1040; CHECK-NEXT: ret1041 %1 = fcmp oeq <8 x bfloat> %a, %b1042 ret <8 x i1> %11043}1044 1045define <8 x i1> @test_fcmp_ogt(<8 x bfloat> %a, <8 x bfloat> %b) #0 {1046; CHECK-LABEL: test_fcmp_ogt:1047; CHECK: // %bb.0:1048; CHECK-NEXT: shll2 v2.4s, v1.8h, #161049; CHECK-NEXT: shll2 v3.4s, v0.8h, #161050; CHECK-NEXT: shll v1.4s, v1.4h, #161051; CHECK-NEXT: shll v0.4s, v0.4h, #161052; CHECK-NEXT: fcmgt v2.4s, v3.4s, v2.4s1053; CHECK-NEXT: fcmgt v0.4s, v0.4s, v1.4s1054; CHECK-NEXT: uzp1 v0.8h, v0.8h, v2.8h1055; CHECK-NEXT: xtn v0.8b, v0.8h1056; CHECK-NEXT: ret1057 %1 = fcmp ogt <8 x bfloat> %a, %b1058 ret <8 x i1> %11059}1060 1061define <8 x i1> @test_fcmp_oge(<8 x bfloat> %a, <8 x bfloat> %b) #0 {1062; CHECK-LABEL: test_fcmp_oge:1063; CHECK: // %bb.0:1064; CHECK-NEXT: shll2 v2.4s, v1.8h, #161065; CHECK-NEXT: shll2 v3.4s, v0.8h, #161066; CHECK-NEXT: shll v1.4s, v1.4h, #161067; CHECK-NEXT: shll v0.4s, v0.4h, #161068; CHECK-NEXT: fcmge v2.4s, v3.4s, v2.4s1069; CHECK-NEXT: fcmge v0.4s, v0.4s, v1.4s1070; CHECK-NEXT: uzp1 v0.8h, v0.8h, v2.8h1071; CHECK-NEXT: xtn v0.8b, v0.8h1072; CHECK-NEXT: ret1073 %1 = fcmp oge <8 x bfloat> %a, %b1074 ret <8 x i1> %11075}1076 1077define <8 x i1> @test_fcmp_olt(<8 x bfloat> %a, <8 x bfloat> %b) #0 {1078; CHECK-LABEL: test_fcmp_olt:1079; CHECK: // %bb.0:1080; CHECK-NEXT: shll2 v2.4s, v0.8h, #161081; CHECK-NEXT: shll2 v3.4s, v1.8h, #161082; CHECK-NEXT: shll v0.4s, v0.4h, #161083; CHECK-NEXT: shll v1.4s, v1.4h, #161084; CHECK-NEXT: fcmgt v2.4s, v3.4s, v2.4s1085; CHECK-NEXT: fcmgt v0.4s, v1.4s, v0.4s1086; CHECK-NEXT: uzp1 v0.8h, v0.8h, v2.8h1087; CHECK-NEXT: xtn v0.8b, v0.8h1088; CHECK-NEXT: ret1089 %1 = fcmp olt <8 x bfloat> %a, %b1090 ret <8 x i1> %11091}1092 1093define <8 x i1> @test_fcmp_ole(<8 x bfloat> %a, <8 x bfloat> %b) #0 {1094; CHECK-LABEL: test_fcmp_ole:1095; CHECK: // %bb.0:1096; CHECK-NEXT: shll2 v2.4s, v0.8h, #161097; CHECK-NEXT: shll2 v3.4s, v1.8h, #161098; CHECK-NEXT: shll v0.4s, v0.4h, #161099; CHECK-NEXT: shll v1.4s, v1.4h, #161100; CHECK-NEXT: fcmge v2.4s, v3.4s, v2.4s1101; CHECK-NEXT: fcmge v0.4s, v1.4s, v0.4s1102; CHECK-NEXT: uzp1 v0.8h, v0.8h, v2.8h1103; CHECK-NEXT: xtn v0.8b, v0.8h1104; CHECK-NEXT: ret1105 %1 = fcmp ole <8 x bfloat> %a, %b1106 ret <8 x i1> %11107}1108 1109define <8 x i1> @test_fcmp_ord(<8 x bfloat> %a, <8 x bfloat> %b) #0 {1110; CHECK-LABEL: test_fcmp_ord:1111; CHECK: // %bb.0:1112; CHECK-NEXT: shll2 v2.4s, v1.8h, #161113; CHECK-NEXT: shll2 v3.4s, v0.8h, #161114; CHECK-NEXT: shll v1.4s, v1.4h, #161115; CHECK-NEXT: shll v0.4s, v0.4h, #161116; CHECK-NEXT: fcmge v4.4s, v3.4s, v2.4s1117; CHECK-NEXT: fcmgt v2.4s, v2.4s, v3.4s1118; CHECK-NEXT: fcmge v3.4s, v0.4s, v1.4s1119; CHECK-NEXT: fcmgt v0.4s, v1.4s, v0.4s1120; CHECK-NEXT: orr v1.16b, v2.16b, v4.16b1121; CHECK-NEXT: orr v0.16b, v0.16b, v3.16b1122; CHECK-NEXT: uzp1 v0.8h, v0.8h, v1.8h1123; CHECK-NEXT: xtn v0.8b, v0.8h1124; CHECK-NEXT: ret1125 %1 = fcmp ord <8 x bfloat> %a, %b1126 ret <8 x i1> %11127}1128 1129attributes #0 = { nounwind }1130