737 lines · cpp
1//2// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.3// See https://llvm.org/LICENSE.txt for license information.4// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception5//6//===----------------------------------------------------------------------===//7//8// This file contains a pass that performs optimization on SIMD instructions9// with high latency by splitting them into more efficient series of10// instructions.11//12// 1. Rewrite certain SIMD instructions with vector element due to their13// inefficiency on some targets.14//15// For example:16// fmla v0.4s, v1.4s, v2.s[1]17//18// Is rewritten into:19// dup v3.4s, v2.s[1]20// fmla v0.4s, v1.4s, v3.4s21//22// 2. Rewrite interleaved memory access instructions due to their23// inefficiency on some targets.24//25// For example:26// st2 {v0.4s, v1.4s}, addr27//28// Is rewritten into:29// zip1 v2.4s, v0.4s, v1.4s30// zip2 v3.4s, v0.4s, v1.4s31// stp q2, q3, addr32//33//===----------------------------------------------------------------------===//34 35#include "AArch64InstrInfo.h"36#include "AArch64Subtarget.h"37#include "llvm/ADT/SmallVector.h"38#include "llvm/ADT/Statistic.h"39#include "llvm/ADT/StringRef.h"40#include "llvm/CodeGen/MachineBasicBlock.h"41#include "llvm/CodeGen/MachineFunction.h"42#include "llvm/CodeGen/MachineFunctionPass.h"43#include "llvm/CodeGen/MachineInstr.h"44#include "llvm/CodeGen/MachineInstrBuilder.h"45#include "llvm/CodeGen/MachineOperand.h"46#include "llvm/CodeGen/MachineRegisterInfo.h"47#include "llvm/CodeGen/TargetInstrInfo.h"48#include "llvm/CodeGen/TargetSchedule.h"49#include "llvm/CodeGen/TargetSubtargetInfo.h"50#include "llvm/MC/MCInstrDesc.h"51#include "llvm/MC/MCSchedule.h"52#include "llvm/Pass.h"53#include <map>54#include <unordered_map>55 56using namespace llvm;57 58#define DEBUG_TYPE "aarch64-simdinstr-opt"59 60STATISTIC(NumModifiedInstr,61 "Number of SIMD instructions modified");62 63#define AARCH64_VECTOR_BY_ELEMENT_OPT_NAME \64 "AArch64 SIMD instructions optimization pass"65 66namespace {67 68struct AArch64SIMDInstrOpt : public MachineFunctionPass {69 static char ID;70 71 const AArch64InstrInfo *TII;72 MachineRegisterInfo *MRI;73 TargetSchedModel SchedModel;74 75 // The two maps below are used to cache decisions instead of recomputing:76 // This is used to cache instruction replacement decisions within function77 // units and across function units.78 std::map<std::pair<unsigned, std::string>, bool> SIMDInstrTable;79 // This is used to cache the decision of whether to leave the interleaved80 // store instructions replacement pass early or not for a particular target.81 std::unordered_map<std::string, bool> InterlEarlyExit;82 83 typedef enum {84 VectorElem,85 Interleave86 } Subpass;87 88 // Instruction represented by OrigOpc is replaced by instructions in ReplOpc.89 struct InstReplInfo {90 unsigned OrigOpc;91 std::vector<unsigned> ReplOpc;92 const TargetRegisterClass RC;93 };94 95#define RuleST2(OpcOrg, OpcR0, OpcR1, OpcR2, RC) \96 {OpcOrg, {OpcR0, OpcR1, OpcR2}, RC}97#define RuleST4(OpcOrg, OpcR0, OpcR1, OpcR2, OpcR3, OpcR4, OpcR5, OpcR6, \98 OpcR7, OpcR8, OpcR9, RC) \99 {OpcOrg, \100 {OpcR0, OpcR1, OpcR2, OpcR3, OpcR4, OpcR5, OpcR6, OpcR7, OpcR8, OpcR9}, RC}101 102 // The Instruction Replacement Table:103 std::vector<InstReplInfo> IRT = {104 // ST2 instructions105 RuleST2(AArch64::ST2Twov2d, AArch64::ZIP1v2i64, AArch64::ZIP2v2i64,106 AArch64::STPQi, AArch64::FPR128RegClass),107 RuleST2(AArch64::ST2Twov4s, AArch64::ZIP1v4i32, AArch64::ZIP2v4i32,108 AArch64::STPQi, AArch64::FPR128RegClass),109 RuleST2(AArch64::ST2Twov2s, AArch64::ZIP1v2i32, AArch64::ZIP2v2i32,110 AArch64::STPDi, AArch64::FPR64RegClass),111 RuleST2(AArch64::ST2Twov8h, AArch64::ZIP1v8i16, AArch64::ZIP2v8i16,112 AArch64::STPQi, AArch64::FPR128RegClass),113 RuleST2(AArch64::ST2Twov4h, AArch64::ZIP1v4i16, AArch64::ZIP2v4i16,114 AArch64::STPDi, AArch64::FPR64RegClass),115 RuleST2(AArch64::ST2Twov16b, AArch64::ZIP1v16i8, AArch64::ZIP2v16i8,116 AArch64::STPQi, AArch64::FPR128RegClass),117 RuleST2(AArch64::ST2Twov8b, AArch64::ZIP1v8i8, AArch64::ZIP2v8i8,118 AArch64::STPDi, AArch64::FPR64RegClass),119 // ST4 instructions120 RuleST4(AArch64::ST4Fourv2d, AArch64::ZIP1v2i64, AArch64::ZIP2v2i64,121 AArch64::ZIP1v2i64, AArch64::ZIP2v2i64, AArch64::ZIP1v2i64,122 AArch64::ZIP2v2i64, AArch64::ZIP1v2i64, AArch64::ZIP2v2i64,123 AArch64::STPQi, AArch64::STPQi, AArch64::FPR128RegClass),124 RuleST4(AArch64::ST4Fourv4s, AArch64::ZIP1v4i32, AArch64::ZIP2v4i32,125 AArch64::ZIP1v4i32, AArch64::ZIP2v4i32, AArch64::ZIP1v4i32,126 AArch64::ZIP2v4i32, AArch64::ZIP1v4i32, AArch64::ZIP2v4i32,127 AArch64::STPQi, AArch64::STPQi, AArch64::FPR128RegClass),128 RuleST4(AArch64::ST4Fourv2s, AArch64::ZIP1v2i32, AArch64::ZIP2v2i32,129 AArch64::ZIP1v2i32, AArch64::ZIP2v2i32, AArch64::ZIP1v2i32,130 AArch64::ZIP2v2i32, AArch64::ZIP1v2i32, AArch64::ZIP2v2i32,131 AArch64::STPDi, AArch64::STPDi, AArch64::FPR64RegClass),132 RuleST4(AArch64::ST4Fourv8h, AArch64::ZIP1v8i16, AArch64::ZIP2v8i16,133 AArch64::ZIP1v8i16, AArch64::ZIP2v8i16, AArch64::ZIP1v8i16,134 AArch64::ZIP2v8i16, AArch64::ZIP1v8i16, AArch64::ZIP2v8i16,135 AArch64::STPQi, AArch64::STPQi, AArch64::FPR128RegClass),136 RuleST4(AArch64::ST4Fourv4h, AArch64::ZIP1v4i16, AArch64::ZIP2v4i16,137 AArch64::ZIP1v4i16, AArch64::ZIP2v4i16, AArch64::ZIP1v4i16,138 AArch64::ZIP2v4i16, AArch64::ZIP1v4i16, AArch64::ZIP2v4i16,139 AArch64::STPDi, AArch64::STPDi, AArch64::FPR64RegClass),140 RuleST4(AArch64::ST4Fourv16b, AArch64::ZIP1v16i8, AArch64::ZIP2v16i8,141 AArch64::ZIP1v16i8, AArch64::ZIP2v16i8, AArch64::ZIP1v16i8,142 AArch64::ZIP2v16i8, AArch64::ZIP1v16i8, AArch64::ZIP2v16i8,143 AArch64::STPQi, AArch64::STPQi, AArch64::FPR128RegClass),144 RuleST4(AArch64::ST4Fourv8b, AArch64::ZIP1v8i8, AArch64::ZIP2v8i8,145 AArch64::ZIP1v8i8, AArch64::ZIP2v8i8, AArch64::ZIP1v8i8,146 AArch64::ZIP2v8i8, AArch64::ZIP1v8i8, AArch64::ZIP2v8i8,147 AArch64::STPDi, AArch64::STPDi, AArch64::FPR64RegClass)148 };149 150 // A costly instruction is replaced in this work by N efficient instructions151 // The maximum of N is currently 10 and it is for ST4 case.152 static const unsigned MaxNumRepl = 10;153 154 AArch64SIMDInstrOpt() : MachineFunctionPass(ID) {}155 156 /// Based only on latency of instructions, determine if it is cost efficient157 /// to replace the instruction InstDesc by the instructions stored in the158 /// array InstDescRepl.159 /// Return true if replacement is expected to be faster.160 bool shouldReplaceInst(MachineFunction *MF, const MCInstrDesc *InstDesc,161 SmallVectorImpl<const MCInstrDesc*> &ReplInstrMCID);162 163 /// Determine if we need to exit the instruction replacement optimization164 /// passes early. This makes sure that no compile time is spent in this pass165 /// for targets with no need for any of these optimizations.166 /// Return true if early exit of the pass is recommended.167 bool shouldExitEarly(MachineFunction *MF, Subpass SP);168 169 /// Check whether an equivalent DUP instruction has already been170 /// created or not.171 /// Return true when the DUP instruction already exists. In this case,172 /// DestReg will point to the destination of the already created DUP.173 bool reuseDUP(MachineInstr &MI, unsigned DupOpcode, unsigned SrcReg,174 unsigned LaneNumber, unsigned *DestReg) const;175 176 /// Certain SIMD instructions with vector element operand are not efficient.177 /// Rewrite them into SIMD instructions with vector operands. This rewrite178 /// is driven by the latency of the instructions.179 /// Return true if the SIMD instruction is modified.180 bool optimizeVectElement(MachineInstr &MI);181 182 /// Process The REG_SEQUENCE instruction, and extract the source183 /// operands of the ST2/4 instruction from it.184 /// Example of such instructions.185 /// %dest = REG_SEQUENCE %st2_src1, dsub0, %st2_src2, dsub1;186 /// Return true when the instruction is processed successfully.187 bool processSeqRegInst(MachineInstr *DefiningMI, unsigned* StReg,188 unsigned* StRegKill, unsigned NumArg) const;189 190 /// Load/Store Interleaving instructions are not always beneficial.191 /// Replace them by ZIP instructionand classical load/store.192 /// Return true if the SIMD instruction is modified.193 bool optimizeLdStInterleave(MachineInstr &MI);194 195 /// Return the number of useful source registers for this196 /// instruction (2 for ST2 and 4 for ST4).197 unsigned determineSrcReg(MachineInstr &MI) const;198 199 bool runOnMachineFunction(MachineFunction &Fn) override;200 201 StringRef getPassName() const override {202 return AARCH64_VECTOR_BY_ELEMENT_OPT_NAME;203 }204};205 206char AArch64SIMDInstrOpt::ID = 0;207 208} // end anonymous namespace209 210INITIALIZE_PASS(AArch64SIMDInstrOpt, "aarch64-simdinstr-opt",211 AARCH64_VECTOR_BY_ELEMENT_OPT_NAME, false, false)212 213/// Based only on latency of instructions, determine if it is cost efficient214/// to replace the instruction InstDesc by the instructions stored in the215/// array InstDescRepl.216/// Return true if replacement is expected to be faster.217bool AArch64SIMDInstrOpt::218shouldReplaceInst(MachineFunction *MF, const MCInstrDesc *InstDesc,219 SmallVectorImpl<const MCInstrDesc*> &InstDescRepl) {220 // Check if replacement decision is already available in the cached table.221 // if so, return it.222 std::string Subtarget = std::string(SchedModel.getSubtargetInfo()->getCPU());223 auto InstID = std::make_pair(InstDesc->getOpcode(), Subtarget);224 auto It = SIMDInstrTable.find(InstID);225 if (It != SIMDInstrTable.end())226 return It->second;227 228 unsigned SCIdx = InstDesc->getSchedClass();229 const MCSchedClassDesc *SCDesc =230 SchedModel.getMCSchedModel()->getSchedClassDesc(SCIdx);231 232 // If a target does not define resources for the instructions233 // of interest, then return false for no replacement.234 const MCSchedClassDesc *SCDescRepl;235 if (!SCDesc->isValid() || SCDesc->isVariant())236 {237 SIMDInstrTable[InstID] = false;238 return false;239 }240 for (const auto *IDesc : InstDescRepl)241 {242 SCDescRepl = SchedModel.getMCSchedModel()->getSchedClassDesc(243 IDesc->getSchedClass());244 if (!SCDescRepl->isValid() || SCDescRepl->isVariant())245 {246 SIMDInstrTable[InstID] = false;247 return false;248 }249 }250 251 // Replacement cost.252 unsigned ReplCost = 0;253 for (const auto *IDesc :InstDescRepl)254 ReplCost += SchedModel.computeInstrLatency(IDesc->getOpcode());255 256 if (SchedModel.computeInstrLatency(InstDesc->getOpcode()) > ReplCost)257 {258 SIMDInstrTable[InstID] = true;259 return true;260 }261 else262 {263 SIMDInstrTable[InstID] = false;264 return false;265 }266}267 268/// Determine if we need to exit this pass for a kind of instruction replacement269/// early. This makes sure that no compile time is spent in this pass for270/// targets with no need for any of these optimizations beyond performing this271/// check.272/// Return true if early exit of this pass for a kind of instruction273/// replacement is recommended for a target.274bool AArch64SIMDInstrOpt::shouldExitEarly(MachineFunction *MF, Subpass SP) {275 const MCInstrDesc* OriginalMCID;276 SmallVector<const MCInstrDesc*, MaxNumRepl> ReplInstrMCID;277 278 switch (SP) {279 // For this optimization, check by comparing the latency of a representative280 // instruction to that of the replacement instructions.281 // TODO: check for all concerned instructions.282 case VectorElem:283 OriginalMCID = &TII->get(AArch64::FMLAv4i32_indexed);284 ReplInstrMCID.push_back(&TII->get(AArch64::DUPv4i32lane));285 ReplInstrMCID.push_back(&TII->get(AArch64::FMLAv4f32));286 if (shouldReplaceInst(MF, OriginalMCID, ReplInstrMCID))287 return false;288 break;289 290 // For this optimization, check for all concerned instructions.291 case Interleave:292 std::string Subtarget =293 std::string(SchedModel.getSubtargetInfo()->getCPU());294 auto It = InterlEarlyExit.find(Subtarget);295 if (It != InterlEarlyExit.end())296 return It->second;297 298 for (auto &I : IRT) {299 OriginalMCID = &TII->get(I.OrigOpc);300 for (auto &Repl : I.ReplOpc)301 ReplInstrMCID.push_back(&TII->get(Repl));302 if (shouldReplaceInst(MF, OriginalMCID, ReplInstrMCID)) {303 InterlEarlyExit[Subtarget] = false;304 return false;305 }306 ReplInstrMCID.clear();307 }308 InterlEarlyExit[Subtarget] = true;309 break;310 }311 312 return true;313}314 315/// Check whether an equivalent DUP instruction has already been316/// created or not.317/// Return true when the DUP instruction already exists. In this case,318/// DestReg will point to the destination of the already created DUP.319bool AArch64SIMDInstrOpt::reuseDUP(MachineInstr &MI, unsigned DupOpcode,320 unsigned SrcReg, unsigned LaneNumber,321 unsigned *DestReg) const {322 for (MachineBasicBlock::iterator MII = MI, MIE = MI.getParent()->begin();323 MII != MIE;) {324 MII--;325 MachineInstr *CurrentMI = &*MII;326 327 if (CurrentMI->getOpcode() == DupOpcode &&328 CurrentMI->getNumOperands() == 3 &&329 CurrentMI->getOperand(1).getReg() == SrcReg &&330 CurrentMI->getOperand(2).getImm() == LaneNumber) {331 *DestReg = CurrentMI->getOperand(0).getReg();332 return true;333 }334 }335 336 return false;337}338 339/// Certain SIMD instructions with vector element operand are not efficient.340/// Rewrite them into SIMD instructions with vector operands. This rewrite341/// is driven by the latency of the instructions.342/// The instruction of concerns are for the time being FMLA, FMLS, FMUL,343/// and FMULX and hence they are hardcoded.344///345/// For example:346/// fmla v0.4s, v1.4s, v2.s[1]347///348/// Is rewritten into349/// dup v3.4s, v2.s[1] // DUP not necessary if redundant350/// fmla v0.4s, v1.4s, v3.4s351///352/// Return true if the SIMD instruction is modified.353bool AArch64SIMDInstrOpt::optimizeVectElement(MachineInstr &MI) {354 const MCInstrDesc *MulMCID, *DupMCID;355 const TargetRegisterClass *RC = &AArch64::FPR128RegClass;356 357 switch (MI.getOpcode()) {358 default:359 return false;360 361 // 4X32 instructions362 case AArch64::FMLAv4i32_indexed:363 DupMCID = &TII->get(AArch64::DUPv4i32lane);364 MulMCID = &TII->get(AArch64::FMLAv4f32);365 break;366 case AArch64::FMLSv4i32_indexed:367 DupMCID = &TII->get(AArch64::DUPv4i32lane);368 MulMCID = &TII->get(AArch64::FMLSv4f32);369 break;370 case AArch64::FMULXv4i32_indexed:371 DupMCID = &TII->get(AArch64::DUPv4i32lane);372 MulMCID = &TII->get(AArch64::FMULXv4f32);373 break;374 case AArch64::FMULv4i32_indexed:375 DupMCID = &TII->get(AArch64::DUPv4i32lane);376 MulMCID = &TII->get(AArch64::FMULv4f32);377 break;378 379 // 2X64 instructions380 case AArch64::FMLAv2i64_indexed:381 DupMCID = &TII->get(AArch64::DUPv2i64lane);382 MulMCID = &TII->get(AArch64::FMLAv2f64);383 break;384 case AArch64::FMLSv2i64_indexed:385 DupMCID = &TII->get(AArch64::DUPv2i64lane);386 MulMCID = &TII->get(AArch64::FMLSv2f64);387 break;388 case AArch64::FMULXv2i64_indexed:389 DupMCID = &TII->get(AArch64::DUPv2i64lane);390 MulMCID = &TII->get(AArch64::FMULXv2f64);391 break;392 case AArch64::FMULv2i64_indexed:393 DupMCID = &TII->get(AArch64::DUPv2i64lane);394 MulMCID = &TII->get(AArch64::FMULv2f64);395 break;396 397 // 2X32 instructions398 case AArch64::FMLAv2i32_indexed:399 RC = &AArch64::FPR64RegClass;400 DupMCID = &TII->get(AArch64::DUPv2i32lane);401 MulMCID = &TII->get(AArch64::FMLAv2f32);402 break;403 case AArch64::FMLSv2i32_indexed:404 RC = &AArch64::FPR64RegClass;405 DupMCID = &TII->get(AArch64::DUPv2i32lane);406 MulMCID = &TII->get(AArch64::FMLSv2f32);407 break;408 case AArch64::FMULXv2i32_indexed:409 RC = &AArch64::FPR64RegClass;410 DupMCID = &TII->get(AArch64::DUPv2i32lane);411 MulMCID = &TII->get(AArch64::FMULXv2f32);412 break;413 case AArch64::FMULv2i32_indexed:414 RC = &AArch64::FPR64RegClass;415 DupMCID = &TII->get(AArch64::DUPv2i32lane);416 MulMCID = &TII->get(AArch64::FMULv2f32);417 break;418 }419 420 SmallVector<const MCInstrDesc*, 2> ReplInstrMCID;421 ReplInstrMCID.push_back(DupMCID);422 ReplInstrMCID.push_back(MulMCID);423 if (!shouldReplaceInst(MI.getParent()->getParent(), &TII->get(MI.getOpcode()),424 ReplInstrMCID))425 return false;426 427 const DebugLoc &DL = MI.getDebugLoc();428 MachineBasicBlock &MBB = *MI.getParent();429 MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo();430 431 // Get the operands of the current SIMD arithmetic instruction.432 Register MulDest = MI.getOperand(0).getReg();433 Register SrcReg0 = MI.getOperand(1).getReg();434 unsigned Src0IsKill = getKillRegState(MI.getOperand(1).isKill());435 Register SrcReg1 = MI.getOperand(2).getReg();436 unsigned Src1IsKill = getKillRegState(MI.getOperand(2).isKill());437 unsigned DupDest;438 439 // Instructions of interest have either 4 or 5 operands.440 if (MI.getNumOperands() == 5) {441 Register SrcReg2 = MI.getOperand(3).getReg();442 unsigned Src2IsKill = getKillRegState(MI.getOperand(3).isKill());443 unsigned LaneNumber = MI.getOperand(4).getImm();444 // Create a new DUP instruction. Note that if an equivalent DUP instruction445 // has already been created before, then use that one instead of creating446 // a new one.447 if (!reuseDUP(MI, DupMCID->getOpcode(), SrcReg2, LaneNumber, &DupDest)) {448 DupDest = MRI.createVirtualRegister(RC);449 BuildMI(MBB, MI, DL, *DupMCID, DupDest)450 .addReg(SrcReg2, Src2IsKill)451 .addImm(LaneNumber);452 }453 BuildMI(MBB, MI, DL, *MulMCID, MulDest)454 .addReg(SrcReg0, Src0IsKill)455 .addReg(SrcReg1, Src1IsKill)456 .addReg(DupDest, Src2IsKill);457 } else if (MI.getNumOperands() == 4) {458 unsigned LaneNumber = MI.getOperand(3).getImm();459 if (!reuseDUP(MI, DupMCID->getOpcode(), SrcReg1, LaneNumber, &DupDest)) {460 DupDest = MRI.createVirtualRegister(RC);461 BuildMI(MBB, MI, DL, *DupMCID, DupDest)462 .addReg(SrcReg1, Src1IsKill)463 .addImm(LaneNumber);464 }465 BuildMI(MBB, MI, DL, *MulMCID, MulDest)466 .addReg(SrcReg0, Src0IsKill)467 .addReg(DupDest, Src1IsKill);468 } else {469 return false;470 }471 472 ++NumModifiedInstr;473 return true;474}475 476/// Load/Store Interleaving instructions are not always beneficial.477/// Replace them by ZIP instructions and classical load/store.478///479/// For example:480/// st2 {v0.4s, v1.4s}, addr481///482/// Is rewritten into:483/// zip1 v2.4s, v0.4s, v1.4s484/// zip2 v3.4s, v0.4s, v1.4s485/// stp q2, q3, addr486//487/// For example:488/// st4 {v0.4s, v1.4s, v2.4s, v3.4s}, addr489///490/// Is rewritten into:491/// zip1 v4.4s, v0.4s, v2.4s492/// zip2 v5.4s, v0.4s, v2.4s493/// zip1 v6.4s, v1.4s, v3.4s494/// zip2 v7.4s, v1.4s, v3.4s495/// zip1 v8.4s, v4.4s, v6.4s496/// zip2 v9.4s, v4.4s, v6.4s497/// zip1 v10.4s, v5.4s, v7.4s498/// zip2 v11.4s, v5.4s, v7.4s499/// stp q8, q9, addr500/// stp q10, q11, addr+32501///502/// Currently only instructions related to ST2 and ST4 are considered.503/// Other may be added later.504/// Return true if the SIMD instruction is modified.505bool AArch64SIMDInstrOpt::optimizeLdStInterleave(MachineInstr &MI) {506 507 unsigned SeqReg, AddrReg;508 unsigned StReg[4], StRegKill[4];509 MachineInstr *DefiningMI;510 const DebugLoc &DL = MI.getDebugLoc();511 MachineBasicBlock &MBB = *MI.getParent();512 SmallVector<unsigned, MaxNumRepl> ZipDest;513 SmallVector<const MCInstrDesc*, MaxNumRepl> ReplInstrMCID;514 515 // If current instruction matches any of the rewriting rules, then516 // gather information about parameters of the new instructions.517 bool Match = false;518 for (auto &I : IRT) {519 if (MI.getOpcode() == I.OrigOpc) {520 SeqReg = MI.getOperand(0).getReg();521 AddrReg = MI.getOperand(1).getReg();522 DefiningMI = MRI->getUniqueVRegDef(SeqReg);523 unsigned NumReg = determineSrcReg(MI);524 if (!processSeqRegInst(DefiningMI, StReg, StRegKill, NumReg))525 return false;526 527 for (auto &Repl : I.ReplOpc) {528 ReplInstrMCID.push_back(&TII->get(Repl));529 // Generate destination registers but only for non-store instruction.530 if (Repl != AArch64::STPQi && Repl != AArch64::STPDi)531 ZipDest.push_back(MRI->createVirtualRegister(&I.RC));532 }533 Match = true;534 break;535 }536 }537 538 if (!Match)539 return false;540 541 // Determine if it is profitable to replace MI by the series of instructions542 // represented in ReplInstrMCID.543 if (!shouldReplaceInst(MI.getParent()->getParent(), &TII->get(MI.getOpcode()),544 ReplInstrMCID))545 return false;546 547 // Generate the replacement instructions composed of ZIP1, ZIP2, and STP (at548 // this point, the code generation is hardcoded and does not rely on the IRT549 // table used above given that code generation for ST2 replacement is somewhat550 // different than for ST4 replacement. We could have added more info into the551 // table related to how we build new instructions but we may be adding more552 // complexity with that).553 switch (MI.getOpcode()) {554 default:555 return false;556 557 case AArch64::ST2Twov16b:558 case AArch64::ST2Twov8b:559 case AArch64::ST2Twov8h:560 case AArch64::ST2Twov4h:561 case AArch64::ST2Twov4s:562 case AArch64::ST2Twov2s:563 case AArch64::ST2Twov2d:564 // ZIP instructions565 BuildMI(MBB, MI, DL, *ReplInstrMCID[0], ZipDest[0])566 .addReg(StReg[0])567 .addReg(StReg[1]);568 BuildMI(MBB, MI, DL, *ReplInstrMCID[1], ZipDest[1])569 .addReg(StReg[0], StRegKill[0])570 .addReg(StReg[1], StRegKill[1]);571 // STP instructions572 BuildMI(MBB, MI, DL, *ReplInstrMCID[2])573 .addReg(ZipDest[0])574 .addReg(ZipDest[1])575 .addReg(AddrReg)576 .addImm(0);577 break;578 579 case AArch64::ST4Fourv16b:580 case AArch64::ST4Fourv8b:581 case AArch64::ST4Fourv8h:582 case AArch64::ST4Fourv4h:583 case AArch64::ST4Fourv4s:584 case AArch64::ST4Fourv2s:585 case AArch64::ST4Fourv2d:586 // ZIP instructions587 BuildMI(MBB, MI, DL, *ReplInstrMCID[0], ZipDest[0])588 .addReg(StReg[0])589 .addReg(StReg[2]);590 BuildMI(MBB, MI, DL, *ReplInstrMCID[1], ZipDest[1])591 .addReg(StReg[0], StRegKill[0])592 .addReg(StReg[2], StRegKill[2]);593 BuildMI(MBB, MI, DL, *ReplInstrMCID[2], ZipDest[2])594 .addReg(StReg[1])595 .addReg(StReg[3]);596 BuildMI(MBB, MI, DL, *ReplInstrMCID[3], ZipDest[3])597 .addReg(StReg[1], StRegKill[1])598 .addReg(StReg[3], StRegKill[3]);599 BuildMI(MBB, MI, DL, *ReplInstrMCID[4], ZipDest[4])600 .addReg(ZipDest[0])601 .addReg(ZipDest[2]);602 BuildMI(MBB, MI, DL, *ReplInstrMCID[5], ZipDest[5])603 .addReg(ZipDest[0])604 .addReg(ZipDest[2]);605 BuildMI(MBB, MI, DL, *ReplInstrMCID[6], ZipDest[6])606 .addReg(ZipDest[1])607 .addReg(ZipDest[3]);608 BuildMI(MBB, MI, DL, *ReplInstrMCID[7], ZipDest[7])609 .addReg(ZipDest[1])610 .addReg(ZipDest[3]);611 // stp instructions612 BuildMI(MBB, MI, DL, *ReplInstrMCID[8])613 .addReg(ZipDest[4])614 .addReg(ZipDest[5])615 .addReg(AddrReg)616 .addImm(0);617 BuildMI(MBB, MI, DL, *ReplInstrMCID[9])618 .addReg(ZipDest[6])619 .addReg(ZipDest[7])620 .addReg(AddrReg)621 .addImm(2);622 break;623 }624 625 ++NumModifiedInstr;626 return true;627}628 629/// Process The REG_SEQUENCE instruction, and extract the source630/// operands of the ST2/4 instruction from it.631/// Example of such instruction.632/// %dest = REG_SEQUENCE %st2_src1, dsub0, %st2_src2, dsub1;633/// Return true when the instruction is processed successfully.634bool AArch64SIMDInstrOpt::processSeqRegInst(MachineInstr *DefiningMI,635 unsigned* StReg, unsigned* StRegKill, unsigned NumArg) const {636 assert(DefiningMI != nullptr);637 if (DefiningMI->getOpcode() != AArch64::REG_SEQUENCE)638 return false;639 640 for (unsigned i=0; i<NumArg; i++) {641 StReg[i] = DefiningMI->getOperand(2*i+1).getReg();642 StRegKill[i] = getKillRegState(DefiningMI->getOperand(2*i+1).isKill());643 644 // Validation check for the other arguments.645 if (DefiningMI->getOperand(2*i+2).isImm()) {646 switch (DefiningMI->getOperand(2*i+2).getImm()) {647 default:648 return false;649 650 case AArch64::dsub0:651 case AArch64::dsub1:652 case AArch64::dsub2:653 case AArch64::dsub3:654 case AArch64::qsub0:655 case AArch64::qsub1:656 case AArch64::qsub2:657 case AArch64::qsub3:658 break;659 }660 }661 else662 return false;663 }664 return true;665}666 667/// Return the number of useful source registers for this instruction668/// (2 for ST2 and 4 for ST4).669unsigned AArch64SIMDInstrOpt::determineSrcReg(MachineInstr &MI) const {670 switch (MI.getOpcode()) {671 default:672 llvm_unreachable("Unsupported instruction for this pass");673 674 case AArch64::ST2Twov16b:675 case AArch64::ST2Twov8b:676 case AArch64::ST2Twov8h:677 case AArch64::ST2Twov4h:678 case AArch64::ST2Twov4s:679 case AArch64::ST2Twov2s:680 case AArch64::ST2Twov2d:681 return 2;682 683 case AArch64::ST4Fourv16b:684 case AArch64::ST4Fourv8b:685 case AArch64::ST4Fourv8h:686 case AArch64::ST4Fourv4h:687 case AArch64::ST4Fourv4s:688 case AArch64::ST4Fourv2s:689 case AArch64::ST4Fourv2d:690 return 4;691 }692}693 694bool AArch64SIMDInstrOpt::runOnMachineFunction(MachineFunction &MF) {695 if (skipFunction(MF.getFunction()))696 return false;697 698 MRI = &MF.getRegInfo();699 const AArch64Subtarget &ST = MF.getSubtarget<AArch64Subtarget>();700 TII = ST.getInstrInfo();701 SchedModel.init(&ST);702 if (!SchedModel.hasInstrSchedModel())703 return false;704 705 bool Changed = false;706 for (auto OptimizationKind : {VectorElem, Interleave}) {707 if (!shouldExitEarly(&MF, OptimizationKind)) {708 SmallVector<MachineInstr *, 8> RemoveMIs;709 for (MachineBasicBlock &MBB : MF) {710 for (MachineInstr &MI : MBB) {711 bool InstRewrite;712 if (OptimizationKind == VectorElem)713 InstRewrite = optimizeVectElement(MI) ;714 else715 InstRewrite = optimizeLdStInterleave(MI);716 if (InstRewrite) {717 // Add MI to the list of instructions to be removed given that it718 // has been replaced.719 RemoveMIs.push_back(&MI);720 Changed = true;721 }722 }723 }724 for (MachineInstr *MI : RemoveMIs)725 MI->eraseFromParent();726 }727 }728 729 return Changed;730}731 732/// Returns an instance of the high cost ASIMD instruction replacement733/// optimization pass.734FunctionPass *llvm::createAArch64SIMDInstrOptPass() {735 return new AArch64SIMDInstrOpt();736}737