519 lines · cpp
1//===- AArch64MacroFusion.cpp - AArch64 Macro Fusion ----------------------===//2//3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.4// See https://llvm.org/LICENSE.txt for license information.5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception6//7//===----------------------------------------------------------------------===//8//9/// \file This file contains the AArch64 implementation of the DAG scheduling10/// mutation to pair instructions back to back.11//12//===----------------------------------------------------------------------===//13 14#include "AArch64MacroFusion.h"15#include "AArch64Subtarget.h"16#include "llvm/CodeGen/MacroFusion.h"17#include "llvm/CodeGen/TargetInstrInfo.h"18 19using namespace llvm;20 21/// CMN, CMP, TST followed by Bcc22static bool isArithmeticBccPair(const MachineInstr *FirstMI,23 const MachineInstr &SecondMI, bool CmpOnly) {24 if (SecondMI.getOpcode() != AArch64::Bcc)25 return false;26 27 // Assume the 1st instr to be a wildcard if it is unspecified.28 if (FirstMI == nullptr)29 return true;30 31 // If we're in CmpOnly mode, we only fuse arithmetic instructions that32 // discard their result.33 if (CmpOnly && FirstMI->getOperand(0).isReg() &&34 !(FirstMI->getOperand(0).getReg() == AArch64::XZR ||35 FirstMI->getOperand(0).getReg() == AArch64::WZR)) {36 return false;37 }38 39 switch (FirstMI->getOpcode()) {40 case AArch64::ADDSWri:41 case AArch64::ADDSWrr:42 case AArch64::ADDSXri:43 case AArch64::ADDSXrr:44 case AArch64::ANDSWri:45 case AArch64::ANDSWrr:46 case AArch64::ANDSXri:47 case AArch64::ANDSXrr:48 case AArch64::SUBSWri:49 case AArch64::SUBSWrr:50 case AArch64::SUBSXri:51 case AArch64::SUBSXrr:52 case AArch64::BICSWrr:53 case AArch64::BICSXrr:54 return true;55 case AArch64::ADDSWrs:56 case AArch64::ADDSXrs:57 case AArch64::ANDSWrs:58 case AArch64::ANDSXrs:59 case AArch64::SUBSWrs:60 case AArch64::SUBSXrs:61 case AArch64::BICSWrs:62 case AArch64::BICSXrs:63 // Shift value can be 0 making these behave like the "rr" variant...64 return !AArch64InstrInfo::hasShiftedReg(*FirstMI);65 }66 67 return false;68}69 70/// ALU operations followed by CBZ/CBNZ.71static bool isArithmeticCbzPair(const MachineInstr *FirstMI,72 const MachineInstr &SecondMI) {73 if (SecondMI.getOpcode() != AArch64::CBZW &&74 SecondMI.getOpcode() != AArch64::CBZX &&75 SecondMI.getOpcode() != AArch64::CBNZW &&76 SecondMI.getOpcode() != AArch64::CBNZX)77 return false;78 79 // Assume the 1st instr to be a wildcard if it is unspecified.80 if (FirstMI == nullptr)81 return true;82 83 switch (FirstMI->getOpcode()) {84 case AArch64::ADDWri:85 case AArch64::ADDWrr:86 case AArch64::ADDXri:87 case AArch64::ADDXrr:88 case AArch64::ANDWri:89 case AArch64::ANDWrr:90 case AArch64::ANDXri:91 case AArch64::ANDXrr:92 case AArch64::EORWri:93 case AArch64::EORWrr:94 case AArch64::EORXri:95 case AArch64::EORXrr:96 case AArch64::ORRWri:97 case AArch64::ORRWrr:98 case AArch64::ORRXri:99 case AArch64::ORRXrr:100 case AArch64::SUBWri:101 case AArch64::SUBWrr:102 case AArch64::SUBXri:103 case AArch64::SUBXrr:104 return true;105 case AArch64::ADDWrs:106 case AArch64::ADDXrs:107 case AArch64::ANDWrs:108 case AArch64::ANDXrs:109 case AArch64::SUBWrs:110 case AArch64::SUBXrs:111 case AArch64::BICWrs:112 case AArch64::BICXrs:113 // Shift value can be 0 making these behave like the "rr" variant...114 return !AArch64InstrInfo::hasShiftedReg(*FirstMI);115 }116 117 return false;118}119 120/// AES crypto encoding or decoding.121static bool isAESPair(const MachineInstr *FirstMI,122 const MachineInstr &SecondMI) {123 // Assume the 1st instr to be a wildcard if it is unspecified.124 switch (SecondMI.getOpcode()) {125 // AES encode.126 case AArch64::AESMCrr:127 case AArch64::AESMCrrTied:128 return FirstMI == nullptr || FirstMI->getOpcode() == AArch64::AESErr;129 // AES decode.130 case AArch64::AESIMCrr:131 case AArch64::AESIMCrrTied:132 return FirstMI == nullptr || FirstMI->getOpcode() == AArch64::AESDrr;133 }134 135 return false;136}137 138/// AESE/AESD/PMULL + EOR.139static bool isCryptoEORPair(const MachineInstr *FirstMI,140 const MachineInstr &SecondMI) {141 if (SecondMI.getOpcode() != AArch64::EORv16i8)142 return false;143 144 // Assume the 1st instr to be a wildcard if it is unspecified.145 if (FirstMI == nullptr)146 return true;147 148 switch (FirstMI->getOpcode()) {149 case AArch64::AESErr:150 case AArch64::AESDrr:151 case AArch64::PMULLv16i8:152 case AArch64::PMULLv8i8:153 case AArch64::PMULLv1i64:154 case AArch64::PMULLv2i64:155 return true;156 }157 158 return false;159}160 161static bool isAdrpAddPair(const MachineInstr *FirstMI,162 const MachineInstr &SecondMI) {163 // Assume the 1st instr to be a wildcard if it is unspecified.164 if ((FirstMI == nullptr || FirstMI->getOpcode() == AArch64::ADRP) &&165 SecondMI.getOpcode() == AArch64::ADDXri)166 return true;167 return false;168}169 170/// Literal generation.171static bool isLiteralsPair(const MachineInstr *FirstMI,172 const MachineInstr &SecondMI) {173 // Assume the 1st instr to be a wildcard if it is unspecified.174 // 32 bit immediate.175 if ((FirstMI == nullptr || FirstMI->getOpcode() == AArch64::MOVZWi) &&176 (SecondMI.getOpcode() == AArch64::MOVKWi &&177 SecondMI.getOperand(3).getImm() == 16))178 return true;179 180 // Lower half of 64 bit immediate.181 if((FirstMI == nullptr || FirstMI->getOpcode() == AArch64::MOVZXi) &&182 (SecondMI.getOpcode() == AArch64::MOVKXi &&183 SecondMI.getOperand(3).getImm() == 16))184 return true;185 186 // Upper half of 64 bit immediate.187 if ((FirstMI == nullptr ||188 (FirstMI->getOpcode() == AArch64::MOVKXi &&189 FirstMI->getOperand(3).getImm() == 32)) &&190 (SecondMI.getOpcode() == AArch64::MOVKXi &&191 SecondMI.getOperand(3).getImm() == 48))192 return true;193 194 return false;195}196 197/// Fuse address generation and loads or stores.198static bool isAddressLdStPair(const MachineInstr *FirstMI,199 const MachineInstr &SecondMI) {200 switch (SecondMI.getOpcode()) {201 case AArch64::STRBBui:202 case AArch64::STRBui:203 case AArch64::STRDui:204 case AArch64::STRHHui:205 case AArch64::STRHui:206 case AArch64::STRQui:207 case AArch64::STRSui:208 case AArch64::STRWui:209 case AArch64::STRXui:210 case AArch64::LDRBBui:211 case AArch64::LDRBui:212 case AArch64::LDRDui:213 case AArch64::LDRHHui:214 case AArch64::LDRHui:215 case AArch64::LDRQui:216 case AArch64::LDRSui:217 case AArch64::LDRWui:218 case AArch64::LDRXui:219 case AArch64::LDRSBWui:220 case AArch64::LDRSBXui:221 case AArch64::LDRSHWui:222 case AArch64::LDRSHXui:223 case AArch64::LDRSWui:224 // Assume the 1st instr to be a wildcard if it is unspecified.225 if (FirstMI == nullptr)226 return true;227 228 switch (FirstMI->getOpcode()) {229 case AArch64::ADR:230 return SecondMI.getOperand(2).getImm() == 0;231 case AArch64::ADRP:232 return true;233 }234 }235 236 return false;237}238 239/// Compare and conditional select.240static bool isCmpCSelPair(const MachineInstr *FirstMI,241 const MachineInstr &SecondMI) {242 // 32 bits243 if (SecondMI.getOpcode() == AArch64::CSELWr) {244 // Assume the 1st instr to be a wildcard if it is unspecified.245 if (FirstMI == nullptr)246 return true;247 248 if (FirstMI->definesRegister(AArch64::WZR, /*TRI=*/nullptr))249 switch (FirstMI->getOpcode()) {250 case AArch64::SUBSWrs:251 return !AArch64InstrInfo::hasShiftedReg(*FirstMI);252 case AArch64::SUBSWrx:253 return !AArch64InstrInfo::hasExtendedReg(*FirstMI);254 case AArch64::SUBSWrr:255 case AArch64::SUBSWri:256 return true;257 }258 }259 260 // 64 bits261 if (SecondMI.getOpcode() == AArch64::CSELXr) {262 // Assume the 1st instr to be a wildcard if it is unspecified.263 if (FirstMI == nullptr)264 return true;265 266 if (FirstMI->definesRegister(AArch64::XZR, /*TRI=*/nullptr))267 switch (FirstMI->getOpcode()) {268 case AArch64::SUBSXrs:269 return !AArch64InstrInfo::hasShiftedReg(*FirstMI);270 case AArch64::SUBSXrx:271 case AArch64::SUBSXrx64:272 return !AArch64InstrInfo::hasExtendedReg(*FirstMI);273 case AArch64::SUBSXrr:274 case AArch64::SUBSXri:275 return true;276 }277 }278 279 return false;280}281 282/// Compare and cset.283static bool isCmpCSetPair(const MachineInstr *FirstMI,284 const MachineInstr &SecondMI) {285 if ((SecondMI.getOpcode() == AArch64::CSINCWr &&286 SecondMI.getOperand(1).getReg() == AArch64::WZR &&287 SecondMI.getOperand(2).getReg() == AArch64::WZR) ||288 (SecondMI.getOpcode() == AArch64::CSINCXr &&289 SecondMI.getOperand(1).getReg() == AArch64::XZR &&290 SecondMI.getOperand(2).getReg() == AArch64::XZR)) {291 // Assume the 1st instr to be a wildcard if it is unspecified.292 if (FirstMI == nullptr)293 return true;294 295 if (FirstMI->definesRegister(AArch64::WZR, /*TRI=*/nullptr) ||296 FirstMI->definesRegister(AArch64::XZR, /*TRI=*/nullptr))297 switch (FirstMI->getOpcode()) {298 case AArch64::SUBSWrs:299 case AArch64::SUBSXrs:300 return !AArch64InstrInfo::hasShiftedReg(*FirstMI);301 case AArch64::SUBSWrx:302 case AArch64::SUBSXrx:303 case AArch64::SUBSXrx64:304 return !AArch64InstrInfo::hasExtendedReg(*FirstMI);305 case AArch64::SUBSWri:306 case AArch64::SUBSWrr:307 case AArch64::SUBSXri:308 case AArch64::SUBSXrr:309 return true;310 }311 }312 313 return false;314}315 316// Arithmetic and logic.317static bool isArithmeticLogicPair(const MachineInstr *FirstMI,318 const MachineInstr &SecondMI) {319 if (AArch64InstrInfo::hasShiftedReg(SecondMI))320 return false;321 322 switch (SecondMI.getOpcode()) {323 // Arithmetic324 case AArch64::ADDWrr:325 case AArch64::ADDXrr:326 case AArch64::SUBWrr:327 case AArch64::SUBXrr:328 case AArch64::ADDWrs:329 case AArch64::ADDXrs:330 case AArch64::SUBWrs:331 case AArch64::SUBXrs:332 // Logic333 case AArch64::ANDWrr:334 case AArch64::ANDXrr:335 case AArch64::BICWrr:336 case AArch64::BICXrr:337 case AArch64::EONWrr:338 case AArch64::EONXrr:339 case AArch64::EORWrr:340 case AArch64::EORXrr:341 case AArch64::ORNWrr:342 case AArch64::ORNXrr:343 case AArch64::ORRWrr:344 case AArch64::ORRXrr:345 case AArch64::ANDWrs:346 case AArch64::ANDXrs:347 case AArch64::BICWrs:348 case AArch64::BICXrs:349 case AArch64::EONWrs:350 case AArch64::EONXrs:351 case AArch64::EORWrs:352 case AArch64::EORXrs:353 case AArch64::ORNWrs:354 case AArch64::ORNXrs:355 case AArch64::ORRWrs:356 case AArch64::ORRXrs:357 // Assume the 1st instr to be a wildcard if it is unspecified.358 if (FirstMI == nullptr)359 return true;360 361 // Arithmetic362 switch (FirstMI->getOpcode()) {363 case AArch64::ADDWrr:364 case AArch64::ADDXrr:365 case AArch64::ADDSWrr:366 case AArch64::ADDSXrr:367 case AArch64::SUBWrr:368 case AArch64::SUBXrr:369 case AArch64::SUBSWrr:370 case AArch64::SUBSXrr:371 return true;372 case AArch64::ADDWrs:373 case AArch64::ADDXrs:374 case AArch64::ADDSWrs:375 case AArch64::ADDSXrs:376 case AArch64::SUBWrs:377 case AArch64::SUBXrs:378 case AArch64::SUBSWrs:379 case AArch64::SUBSXrs:380 return !AArch64InstrInfo::hasShiftedReg(*FirstMI);381 }382 break;383 384 // Arithmetic, setting flags.385 case AArch64::ADDSWrr:386 case AArch64::ADDSXrr:387 case AArch64::SUBSWrr:388 case AArch64::SUBSXrr:389 case AArch64::ADDSWrs:390 case AArch64::ADDSXrs:391 case AArch64::SUBSWrs:392 case AArch64::SUBSXrs:393 // Assume the 1st instr to be a wildcard if it is unspecified.394 if (FirstMI == nullptr)395 return true;396 397 // Arithmetic, not setting flags.398 switch (FirstMI->getOpcode()) {399 case AArch64::ADDWrr:400 case AArch64::ADDXrr:401 case AArch64::SUBWrr:402 case AArch64::SUBXrr:403 return true;404 case AArch64::ADDWrs:405 case AArch64::ADDXrs:406 case AArch64::SUBWrs:407 case AArch64::SUBXrs:408 return !AArch64InstrInfo::hasShiftedReg(*FirstMI);409 }410 break;411 }412 413 return false;414}415 416// "(A + B) + 1" or "(A - B) - 1"417static bool isAddSub2RegAndConstOnePair(const MachineInstr *FirstMI,418 const MachineInstr &SecondMI) {419 bool NeedsSubtract = false;420 421 // The 2nd instr must be an add-immediate or subtract-immediate.422 switch (SecondMI.getOpcode()) {423 case AArch64::SUBWri:424 case AArch64::SUBXri:425 NeedsSubtract = true;426 [[fallthrough]];427 case AArch64::ADDWri:428 case AArch64::ADDXri:429 break;430 431 default:432 return false;433 }434 435 // The immediate in the 2nd instr must be "1".436 if (!SecondMI.getOperand(2).isImm() || SecondMI.getOperand(2).getImm() != 1) {437 return false;438 }439 440 // Assume the 1st instr to be a wildcard if it is unspecified.441 if (FirstMI == nullptr) {442 return true;443 }444 445 switch (FirstMI->getOpcode()) {446 case AArch64::SUBWrs:447 case AArch64::SUBXrs:448 if (AArch64InstrInfo::hasShiftedReg(*FirstMI))449 return false;450 [[fallthrough]];451 case AArch64::SUBWrr:452 case AArch64::SUBXrr:453 if (NeedsSubtract) {454 return true;455 }456 break;457 458 case AArch64::ADDWrs:459 case AArch64::ADDXrs:460 if (AArch64InstrInfo::hasShiftedReg(*FirstMI))461 return false;462 [[fallthrough]];463 case AArch64::ADDWrr:464 case AArch64::ADDXrr:465 if (!NeedsSubtract) {466 return true;467 }468 break;469 }470 471 return false;472}473 474/// \brief Check if the instr pair, FirstMI and SecondMI, should be fused475/// together. Given SecondMI, when FirstMI is unspecified, then check if476/// SecondMI may be part of a fused pair at all.477static bool shouldScheduleAdjacent(const TargetInstrInfo &TII,478 const TargetSubtargetInfo &TSI,479 const MachineInstr *FirstMI,480 const MachineInstr &SecondMI) {481 const AArch64Subtarget &ST = static_cast<const AArch64Subtarget&>(TSI);482 483 // All checking functions assume that the 1st instr is a wildcard if it is484 // unspecified.485 if (ST.hasCmpBccFusion() || ST.hasArithmeticBccFusion()) {486 bool CmpOnly = !ST.hasArithmeticBccFusion();487 if (isArithmeticBccPair(FirstMI, SecondMI, CmpOnly))488 return true;489 }490 if (ST.hasArithmeticCbzFusion() && isArithmeticCbzPair(FirstMI, SecondMI))491 return true;492 if (ST.hasFuseAES() && isAESPair(FirstMI, SecondMI))493 return true;494 if (ST.hasFuseCryptoEOR() && isCryptoEORPair(FirstMI, SecondMI))495 return true;496 if (ST.hasFuseAdrpAdd() && isAdrpAddPair(FirstMI, SecondMI))497 return true;498 if (ST.hasFuseLiterals() && isLiteralsPair(FirstMI, SecondMI))499 return true;500 if (ST.hasFuseAddress() && isAddressLdStPair(FirstMI, SecondMI))501 return true;502 if (ST.hasFuseCmpCSel() && isCmpCSelPair(FirstMI, SecondMI))503 return true;504 if (ST.hasFuseCmpCSet() && isCmpCSetPair(FirstMI, SecondMI))505 return true;506 if (ST.hasFuseArithmeticLogic() && isArithmeticLogicPair(FirstMI, SecondMI))507 return true;508 if (ST.hasFuseAddSub2RegAndConstOne() &&509 isAddSub2RegAndConstOnePair(FirstMI, SecondMI))510 return true;511 512 return false;513}514 515std::unique_ptr<ScheduleDAGMutation>516llvm::createAArch64MacroFusionDAGMutation() {517 return createMacroFusionDAGMutation(shouldScheduleAdjacent);518}519