866 lines · cpp
1//===---- CGLoopInfo.cpp - LLVM CodeGen for loop metadata -*- C++ -*-------===//2//3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.4// See https://llvm.org/LICENSE.txt for license information.5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception6//7//===----------------------------------------------------------------------===//8 9#include "CGLoopInfo.h"10#include "clang/AST/ASTContext.h"11#include "clang/AST/Attr.h"12#include "clang/AST/Expr.h"13#include "clang/Basic/CodeGenOptions.h"14#include "llvm/IR/BasicBlock.h"15#include "llvm/IR/CFG.h"16#include "llvm/IR/Constants.h"17#include "llvm/IR/InstrTypes.h"18#include "llvm/IR/Instructions.h"19#include "llvm/IR/Metadata.h"20#include <optional>21using namespace clang::CodeGen;22using namespace llvm;23 24MDNode *25LoopInfo::createFollowupMetadata(const char *FollowupName,26 ArrayRef<llvm::Metadata *> LoopProperties) {27 LLVMContext &Ctx = Header->getContext();28 29 SmallVector<Metadata *, 4> Args;30 Args.push_back(MDString::get(Ctx, FollowupName));31 Args.append(LoopProperties.begin(), LoopProperties.end());32 return MDNode::get(Ctx, Args);33}34 35SmallVector<Metadata *, 4>36LoopInfo::createPipeliningMetadata(const LoopAttributes &Attrs,37 ArrayRef<Metadata *> LoopProperties,38 bool &HasUserTransforms) {39 LLVMContext &Ctx = Header->getContext();40 41 std::optional<bool> Enabled;42 if (Attrs.PipelineDisabled)43 Enabled = false;44 else if (Attrs.PipelineInitiationInterval != 0)45 Enabled = true;46 47 SmallVector<Metadata *, 4> Args;48 Args.append(LoopProperties.begin(), LoopProperties.end());49 50 if (Enabled != true) {51 if (Enabled == false) {52 Args.push_back(53 MDNode::get(Ctx, {MDString::get(Ctx, "llvm.loop.pipeline.disable"),54 ConstantAsMetadata::get(ConstantInt::get(55 llvm::Type::getInt1Ty(Ctx), 1))}));56 }57 return Args;58 }59 60 if (Attrs.PipelineInitiationInterval > 0) {61 Metadata *Vals[] = {62 MDString::get(Ctx, "llvm.loop.pipeline.initiationinterval"),63 ConstantAsMetadata::get(ConstantInt::get(64 llvm::Type::getInt32Ty(Ctx), Attrs.PipelineInitiationInterval))};65 Args.push_back(MDNode::get(Ctx, Vals));66 }67 68 // No follow-up: This is the last transformation.69 70 HasUserTransforms = true;71 return Args;72}73 74SmallVector<Metadata *, 4>75LoopInfo::createPartialUnrollMetadata(const LoopAttributes &Attrs,76 ArrayRef<Metadata *> LoopProperties,77 bool &HasUserTransforms) {78 LLVMContext &Ctx = Header->getContext();79 80 std::optional<bool> Enabled;81 if (Attrs.UnrollEnable == LoopAttributes::Disable)82 Enabled = false;83 else if (Attrs.UnrollEnable == LoopAttributes::Full)84 Enabled = std::nullopt;85 else if (Attrs.UnrollEnable != LoopAttributes::Unspecified ||86 Attrs.UnrollCount != 0)87 Enabled = true;88 89 if (Enabled != true) {90 // createFullUnrollMetadata will already have added llvm.loop.unroll.disable91 // if unrolling is disabled.92 return createPipeliningMetadata(Attrs, LoopProperties, HasUserTransforms);93 }94 95 SmallVector<Metadata *, 4> FollowupLoopProperties;96 97 // Apply all loop properties to the unrolled loop.98 FollowupLoopProperties.append(LoopProperties.begin(), LoopProperties.end());99 100 // Don't unroll an already unrolled loop.101 FollowupLoopProperties.push_back(102 MDNode::get(Ctx, MDString::get(Ctx, "llvm.loop.unroll.disable")));103 104 bool FollowupHasTransforms = false;105 SmallVector<Metadata *, 4> Followup = createPipeliningMetadata(106 Attrs, FollowupLoopProperties, FollowupHasTransforms);107 108 SmallVector<Metadata *, 4> Args;109 Args.append(LoopProperties.begin(), LoopProperties.end());110 111 // Setting unroll.count112 if (Attrs.UnrollCount > 0) {113 Metadata *Vals[] = {MDString::get(Ctx, "llvm.loop.unroll.count"),114 ConstantAsMetadata::get(ConstantInt::get(115 llvm::Type::getInt32Ty(Ctx), Attrs.UnrollCount))};116 Args.push_back(MDNode::get(Ctx, Vals));117 }118 119 // Setting unroll.full or unroll.disable120 if (Attrs.UnrollEnable == LoopAttributes::Enable) {121 Metadata *Vals[] = {MDString::get(Ctx, "llvm.loop.unroll.enable")};122 Args.push_back(MDNode::get(Ctx, Vals));123 }124 125 if (FollowupHasTransforms)126 Args.push_back(127 createFollowupMetadata("llvm.loop.unroll.followup_all", Followup));128 129 HasUserTransforms = true;130 return Args;131}132 133SmallVector<Metadata *, 4>134LoopInfo::createUnrollAndJamMetadata(const LoopAttributes &Attrs,135 ArrayRef<Metadata *> LoopProperties,136 bool &HasUserTransforms) {137 LLVMContext &Ctx = Header->getContext();138 139 std::optional<bool> Enabled;140 if (Attrs.UnrollAndJamEnable == LoopAttributes::Disable)141 Enabled = false;142 else if (Attrs.UnrollAndJamEnable == LoopAttributes::Enable ||143 Attrs.UnrollAndJamCount != 0)144 Enabled = true;145 146 if (Enabled != true) {147 SmallVector<Metadata *, 4> NewLoopProperties;148 if (Enabled == false) {149 NewLoopProperties.append(LoopProperties.begin(), LoopProperties.end());150 NewLoopProperties.push_back(MDNode::get(151 Ctx, MDString::get(Ctx, "llvm.loop.unroll_and_jam.disable")));152 LoopProperties = NewLoopProperties;153 }154 return createPartialUnrollMetadata(Attrs, LoopProperties,155 HasUserTransforms);156 }157 158 SmallVector<Metadata *, 4> FollowupLoopProperties;159 FollowupLoopProperties.append(LoopProperties.begin(), LoopProperties.end());160 FollowupLoopProperties.push_back(161 MDNode::get(Ctx, MDString::get(Ctx, "llvm.loop.unroll_and_jam.disable")));162 163 bool FollowupHasTransforms = false;164 SmallVector<Metadata *, 4> Followup = createPartialUnrollMetadata(165 Attrs, FollowupLoopProperties, FollowupHasTransforms);166 167 SmallVector<Metadata *, 4> Args;168 Args.append(LoopProperties.begin(), LoopProperties.end());169 170 // Setting unroll_and_jam.count171 if (Attrs.UnrollAndJamCount > 0) {172 Metadata *Vals[] = {173 MDString::get(Ctx, "llvm.loop.unroll_and_jam.count"),174 ConstantAsMetadata::get(ConstantInt::get(llvm::Type::getInt32Ty(Ctx),175 Attrs.UnrollAndJamCount))};176 Args.push_back(MDNode::get(Ctx, Vals));177 }178 179 if (Attrs.UnrollAndJamEnable == LoopAttributes::Enable) {180 Metadata *Vals[] = {MDString::get(Ctx, "llvm.loop.unroll_and_jam.enable")};181 Args.push_back(MDNode::get(Ctx, Vals));182 }183 184 if (FollowupHasTransforms)185 Args.push_back(createFollowupMetadata(186 "llvm.loop.unroll_and_jam.followup_outer", Followup));187 188 if (UnrollAndJamInnerFollowup.has_value())189 Args.push_back(createFollowupMetadata(190 "llvm.loop.unroll_and_jam.followup_inner", *UnrollAndJamInnerFollowup));191 192 HasUserTransforms = true;193 return Args;194}195 196SmallVector<Metadata *, 4>197LoopInfo::createLoopVectorizeMetadata(const LoopAttributes &Attrs,198 ArrayRef<Metadata *> LoopProperties,199 bool &HasUserTransforms) {200 LLVMContext &Ctx = Header->getContext();201 202 std::optional<bool> Enabled;203 if (Attrs.VectorizeEnable == LoopAttributes::Disable)204 Enabled = false;205 else if (Attrs.VectorizeEnable != LoopAttributes::Unspecified ||206 Attrs.VectorizePredicateEnable != LoopAttributes::Unspecified ||207 Attrs.InterleaveCount != 0 || Attrs.VectorizeWidth != 0 ||208 Attrs.VectorizeScalable != LoopAttributes::Unspecified)209 Enabled = true;210 211 if (Enabled != true) {212 SmallVector<Metadata *, 4> NewLoopProperties;213 if (Enabled == false) {214 NewLoopProperties.append(LoopProperties.begin(), LoopProperties.end());215 NewLoopProperties.push_back(216 MDNode::get(Ctx, {MDString::get(Ctx, "llvm.loop.vectorize.enable"),217 ConstantAsMetadata::get(ConstantInt::get(218 llvm::Type::getInt1Ty(Ctx), 0))}));219 LoopProperties = NewLoopProperties;220 }221 return createUnrollAndJamMetadata(Attrs, LoopProperties, HasUserTransforms);222 }223 224 SmallVector<Metadata *, 4> Args;225 Args.append(LoopProperties.begin(), LoopProperties.end());226 227 // Setting vectorize.predicate when it has been specified and vectorization228 // has not been disabled.229 bool IsVectorPredicateEnabled = false;230 if (Attrs.VectorizePredicateEnable != LoopAttributes::Unspecified) {231 IsVectorPredicateEnabled =232 (Attrs.VectorizePredicateEnable == LoopAttributes::Enable);233 234 Metadata *Vals[] = {235 MDString::get(Ctx, "llvm.loop.vectorize.predicate.enable"),236 ConstantAsMetadata::get(ConstantInt::get(llvm::Type::getInt1Ty(Ctx),237 IsVectorPredicateEnabled))};238 Args.push_back(MDNode::get(Ctx, Vals));239 }240 241 // Setting vectorize.width242 if (Attrs.VectorizeWidth > 0) {243 Metadata *Vals[] = {244 MDString::get(Ctx, "llvm.loop.vectorize.width"),245 ConstantAsMetadata::get(ConstantInt::get(llvm::Type::getInt32Ty(Ctx),246 Attrs.VectorizeWidth))};247 248 Args.push_back(MDNode::get(Ctx, Vals));249 }250 251 if (Attrs.VectorizeScalable != LoopAttributes::Unspecified) {252 bool IsScalable = Attrs.VectorizeScalable == LoopAttributes::Enable;253 Metadata *Vals[] = {254 MDString::get(Ctx, "llvm.loop.vectorize.scalable.enable"),255 ConstantAsMetadata::get(256 ConstantInt::get(llvm::Type::getInt1Ty(Ctx), IsScalable))};257 Args.push_back(MDNode::get(Ctx, Vals));258 }259 260 // Setting interleave.count261 if (Attrs.InterleaveCount > 0) {262 Metadata *Vals[] = {263 MDString::get(Ctx, "llvm.loop.interleave.count"),264 ConstantAsMetadata::get(ConstantInt::get(llvm::Type::getInt32Ty(Ctx),265 Attrs.InterleaveCount))};266 Args.push_back(MDNode::get(Ctx, Vals));267 }268 269 // vectorize.enable is set if:270 // 1) loop hint vectorize.enable is set, or271 // 2) it is implied when vectorize.predicate is set, or272 // 3) it is implied when vectorize.width is set to a value > 1273 // 4) it is implied when vectorize.scalable.enable is true274 // 5) it is implied when vectorize.width is unset (0) and the user275 // explicitly requested fixed-width vectorization, i.e.276 // vectorize.scalable.enable is false.277 bool VectorizeEnabled = false;278 if (Attrs.VectorizeEnable != LoopAttributes::Unspecified ||279 (IsVectorPredicateEnabled && Attrs.VectorizeWidth != 1) ||280 Attrs.VectorizeWidth > 1 ||281 Attrs.VectorizeScalable == LoopAttributes::Enable ||282 (Attrs.VectorizeScalable == LoopAttributes::Disable &&283 Attrs.VectorizeWidth != 1)) {284 VectorizeEnabled = Attrs.VectorizeEnable != LoopAttributes::Disable;285 Args.push_back(286 MDNode::get(Ctx, {MDString::get(Ctx, "llvm.loop.vectorize.enable"),287 ConstantAsMetadata::get(ConstantInt::get(288 llvm::Type::getInt1Ty(Ctx), VectorizeEnabled))}));289 }290 291 // Apply all loop properties to the vectorized loop.292 SmallVector<Metadata *, 4> FollowupLoopProperties;293 294 // If vectorization is not explicitly enabled, the follow-up metadata will be295 // directly appended to the list currently being created. In that case, adding296 // LoopProperties to FollowupLoopProperties would result in duplication.297 if (VectorizeEnabled)298 FollowupLoopProperties.append(LoopProperties.begin(), LoopProperties.end());299 300 // Don't vectorize an already vectorized loop.301 FollowupLoopProperties.push_back(302 MDNode::get(Ctx, MDString::get(Ctx, "llvm.loop.isvectorized")));303 304 bool FollowupHasTransforms = false;305 SmallVector<Metadata *, 4> Followup = createUnrollAndJamMetadata(306 Attrs, FollowupLoopProperties, FollowupHasTransforms);307 308 if (FollowupHasTransforms) {309 // If vectorization is explicitly enabled, we create a follow-up metadata,310 // otherwise directly add the contents of it to Args.311 if (VectorizeEnabled)312 Args.push_back(313 createFollowupMetadata("llvm.loop.vectorize.followup_all", Followup));314 else315 Args.append(Followup.begin(), Followup.end());316 }317 318 HasUserTransforms = true;319 return Args;320}321 322SmallVector<Metadata *, 4>323LoopInfo::createLoopDistributeMetadata(const LoopAttributes &Attrs,324 ArrayRef<Metadata *> LoopProperties,325 bool &HasUserTransforms) {326 LLVMContext &Ctx = Header->getContext();327 328 std::optional<bool> Enabled;329 if (Attrs.DistributeEnable == LoopAttributes::Disable)330 Enabled = false;331 if (Attrs.DistributeEnable == LoopAttributes::Enable)332 Enabled = true;333 334 if (Enabled != true) {335 SmallVector<Metadata *, 4> NewLoopProperties;336 if (Enabled == false) {337 NewLoopProperties.append(LoopProperties.begin(), LoopProperties.end());338 NewLoopProperties.push_back(339 MDNode::get(Ctx, {MDString::get(Ctx, "llvm.loop.distribute.enable"),340 ConstantAsMetadata::get(ConstantInt::get(341 llvm::Type::getInt1Ty(Ctx), 0))}));342 LoopProperties = NewLoopProperties;343 }344 return createLoopVectorizeMetadata(Attrs, LoopProperties,345 HasUserTransforms);346 }347 348 bool FollowupHasTransforms = false;349 SmallVector<Metadata *, 4> Followup =350 createLoopVectorizeMetadata(Attrs, LoopProperties, FollowupHasTransforms);351 352 SmallVector<Metadata *, 4> Args;353 Args.append(LoopProperties.begin(), LoopProperties.end());354 355 Metadata *Vals[] = {MDString::get(Ctx, "llvm.loop.distribute.enable"),356 ConstantAsMetadata::get(ConstantInt::get(357 llvm::Type::getInt1Ty(Ctx),358 (Attrs.DistributeEnable == LoopAttributes::Enable)))};359 Args.push_back(MDNode::get(Ctx, Vals));360 361 if (FollowupHasTransforms)362 Args.push_back(363 createFollowupMetadata("llvm.loop.distribute.followup_all", Followup));364 365 HasUserTransforms = true;366 return Args;367}368 369SmallVector<Metadata *, 4>370LoopInfo::createFullUnrollMetadata(const LoopAttributes &Attrs,371 ArrayRef<Metadata *> LoopProperties,372 bool &HasUserTransforms) {373 LLVMContext &Ctx = Header->getContext();374 375 std::optional<bool> Enabled;376 if (Attrs.UnrollEnable == LoopAttributes::Disable)377 Enabled = false;378 else if (Attrs.UnrollEnable == LoopAttributes::Full)379 Enabled = true;380 381 if (Enabled != true) {382 SmallVector<Metadata *, 4> NewLoopProperties;383 if (Enabled == false) {384 NewLoopProperties.append(LoopProperties.begin(), LoopProperties.end());385 NewLoopProperties.push_back(386 MDNode::get(Ctx, MDString::get(Ctx, "llvm.loop.unroll.disable")));387 LoopProperties = NewLoopProperties;388 }389 return createLoopDistributeMetadata(Attrs, LoopProperties,390 HasUserTransforms);391 }392 393 SmallVector<Metadata *, 4> Args;394 Args.append(LoopProperties.begin(), LoopProperties.end());395 Args.push_back(MDNode::get(Ctx, MDString::get(Ctx, "llvm.loop.unroll.full")));396 397 // No follow-up: there is no loop after full unrolling.398 // TODO: Warn if there are transformations after full unrolling.399 400 HasUserTransforms = true;401 return Args;402}403 404SmallVector<Metadata *, 4> LoopInfo::createMetadata(405 const LoopAttributes &Attrs,406 llvm::ArrayRef<llvm::Metadata *> AdditionalLoopProperties,407 bool &HasUserTransforms) {408 SmallVector<Metadata *, 3> LoopProperties;409 410 // If we have a valid start debug location for the loop, add it.411 if (StartLoc) {412 LoopProperties.push_back(StartLoc.getAsMDNode());413 414 // If we also have a valid end debug location for the loop, add it.415 if (EndLoc)416 LoopProperties.push_back(EndLoc.getAsMDNode());417 }418 419 LLVMContext &Ctx = Header->getContext();420 if (Attrs.MustProgress)421 LoopProperties.push_back(422 MDNode::get(Ctx, MDString::get(Ctx, "llvm.loop.mustprogress")));423 424 assert(!!AccGroup == Attrs.IsParallel &&425 "There must be an access group iff the loop is parallel");426 if (Attrs.IsParallel) {427 LoopProperties.push_back(MDNode::get(428 Ctx, {MDString::get(Ctx, "llvm.loop.parallel_accesses"), AccGroup}));429 }430 431 // Setting clang::code_align attribute.432 if (Attrs.CodeAlign > 0) {433 Metadata *Vals[] = {MDString::get(Ctx, "llvm.loop.align"),434 ConstantAsMetadata::get(ConstantInt::get(435 llvm::Type::getInt32Ty(Ctx), Attrs.CodeAlign))};436 LoopProperties.push_back(MDNode::get(Ctx, Vals));437 }438 439 llvm::append_range(LoopProperties, AdditionalLoopProperties);440 return createFullUnrollMetadata(Attrs, LoopProperties, HasUserTransforms);441}442 443LoopAttributes::LoopAttributes(bool IsParallel)444 : IsParallel(IsParallel), VectorizeEnable(LoopAttributes::Unspecified),445 UnrollEnable(LoopAttributes::Unspecified),446 UnrollAndJamEnable(LoopAttributes::Unspecified),447 VectorizePredicateEnable(LoopAttributes::Unspecified), VectorizeWidth(0),448 VectorizeScalable(LoopAttributes::Unspecified), InterleaveCount(0),449 UnrollCount(0), UnrollAndJamCount(0),450 DistributeEnable(LoopAttributes::Unspecified), PipelineDisabled(false),451 PipelineInitiationInterval(0), CodeAlign(0), MustProgress(false) {}452 453void LoopAttributes::clear() {454 IsParallel = false;455 VectorizeWidth = 0;456 VectorizeScalable = LoopAttributes::Unspecified;457 InterleaveCount = 0;458 UnrollCount = 0;459 UnrollAndJamCount = 0;460 VectorizeEnable = LoopAttributes::Unspecified;461 UnrollEnable = LoopAttributes::Unspecified;462 UnrollAndJamEnable = LoopAttributes::Unspecified;463 VectorizePredicateEnable = LoopAttributes::Unspecified;464 DistributeEnable = LoopAttributes::Unspecified;465 PipelineDisabled = false;466 PipelineInitiationInterval = 0;467 CodeAlign = 0;468 MustProgress = false;469}470 471LoopInfo::LoopInfo(BasicBlock *Header, const LoopAttributes &Attrs,472 const llvm::DebugLoc &StartLoc, const llvm::DebugLoc &EndLoc,473 LoopInfo *Parent)474 : Header(Header), Attrs(Attrs), StartLoc(StartLoc), EndLoc(EndLoc),475 Parent(Parent) {476 477 if (Attrs.IsParallel) {478 // Create an access group for this loop.479 LLVMContext &Ctx = Header->getContext();480 AccGroup = MDNode::getDistinct(Ctx, {});481 }482 483 if (!Attrs.IsParallel && Attrs.VectorizeWidth == 0 &&484 Attrs.VectorizeScalable == LoopAttributes::Unspecified &&485 Attrs.InterleaveCount == 0 && Attrs.UnrollCount == 0 &&486 Attrs.UnrollAndJamCount == 0 && !Attrs.PipelineDisabled &&487 Attrs.PipelineInitiationInterval == 0 &&488 Attrs.VectorizePredicateEnable == LoopAttributes::Unspecified &&489 Attrs.VectorizeEnable == LoopAttributes::Unspecified &&490 Attrs.UnrollEnable == LoopAttributes::Unspecified &&491 Attrs.UnrollAndJamEnable == LoopAttributes::Unspecified &&492 Attrs.DistributeEnable == LoopAttributes::Unspecified &&493 Attrs.CodeAlign == 0 && !StartLoc && !EndLoc && !Attrs.MustProgress)494 return;495 496 TempLoopID = MDNode::getTemporary(Header->getContext(), {});497}498 499void LoopInfo::finish() {500 // We did not annotate the loop body instructions because there are no501 // attributes for this loop.502 if (!TempLoopID)503 return;504 505 MDNode *LoopID;506 LoopAttributes CurLoopAttr = Attrs;507 LLVMContext &Ctx = Header->getContext();508 509 if (Parent && (Parent->Attrs.UnrollAndJamEnable ||510 Parent->Attrs.UnrollAndJamCount != 0)) {511 // Parent unroll-and-jams this loop.512 // Split the transformations in those that happens before the unroll-and-jam513 // and those after.514 515 LoopAttributes BeforeJam, AfterJam;516 517 BeforeJam.IsParallel = AfterJam.IsParallel = Attrs.IsParallel;518 519 BeforeJam.VectorizeWidth = Attrs.VectorizeWidth;520 BeforeJam.VectorizeScalable = Attrs.VectorizeScalable;521 BeforeJam.InterleaveCount = Attrs.InterleaveCount;522 BeforeJam.VectorizeEnable = Attrs.VectorizeEnable;523 BeforeJam.DistributeEnable = Attrs.DistributeEnable;524 BeforeJam.VectorizePredicateEnable = Attrs.VectorizePredicateEnable;525 526 switch (Attrs.UnrollEnable) {527 case LoopAttributes::Unspecified:528 case LoopAttributes::Disable:529 BeforeJam.UnrollEnable = Attrs.UnrollEnable;530 AfterJam.UnrollEnable = Attrs.UnrollEnable;531 break;532 case LoopAttributes::Full:533 BeforeJam.UnrollEnable = LoopAttributes::Full;534 break;535 case LoopAttributes::Enable:536 AfterJam.UnrollEnable = LoopAttributes::Enable;537 break;538 }539 540 AfterJam.VectorizePredicateEnable = Attrs.VectorizePredicateEnable;541 AfterJam.UnrollCount = Attrs.UnrollCount;542 AfterJam.PipelineDisabled = Attrs.PipelineDisabled;543 AfterJam.PipelineInitiationInterval = Attrs.PipelineInitiationInterval;544 545 // If this loop is subject of an unroll-and-jam by the parent loop, and has546 // an unroll-and-jam annotation itself, we have to decide whether to first547 // apply the parent's unroll-and-jam or this loop's unroll-and-jam. The548 // UnrollAndJam pass processes loops from inner to outer, so we apply the549 // inner first.550 BeforeJam.UnrollAndJamCount = Attrs.UnrollAndJamCount;551 BeforeJam.UnrollAndJamEnable = Attrs.UnrollAndJamEnable;552 553 // Set the inner followup metadata to process by the outer loop. Only554 // consider the first inner loop.555 if (!Parent->UnrollAndJamInnerFollowup) {556 // Splitting the attributes into a BeforeJam and an AfterJam part will557 // stop 'llvm.loop.isvectorized' (generated by vectorization in BeforeJam)558 // to be forwarded to the AfterJam part. We detect the situation here and559 // add it manually.560 SmallVector<Metadata *, 1> BeforeLoopProperties;561 if (BeforeJam.VectorizeEnable != LoopAttributes::Unspecified ||562 BeforeJam.VectorizePredicateEnable != LoopAttributes::Unspecified ||563 BeforeJam.InterleaveCount != 0 || BeforeJam.VectorizeWidth != 0 ||564 BeforeJam.VectorizeScalable == LoopAttributes::Enable)565 BeforeLoopProperties.push_back(566 MDNode::get(Ctx, MDString::get(Ctx, "llvm.loop.isvectorized")));567 568 bool InnerFollowupHasTransform = false;569 SmallVector<Metadata *, 4> InnerFollowup = createMetadata(570 AfterJam, BeforeLoopProperties, InnerFollowupHasTransform);571 if (InnerFollowupHasTransform)572 Parent->UnrollAndJamInnerFollowup = InnerFollowup;573 }574 575 CurLoopAttr = BeforeJam;576 }577 578 bool HasUserTransforms = false;579 SmallVector<Metadata *, 4> Properties =580 createMetadata(CurLoopAttr, {}, HasUserTransforms);581 SmallVector<Metadata *, 4> Args;582 Args.push_back(nullptr);583 Args.append(Properties.begin(), Properties.end());584 LoopID = MDNode::getDistinct(Ctx, Args);585 LoopID->replaceOperandWith(0, LoopID);586 587 TempLoopID->replaceAllUsesWith(LoopID);588}589 590void LoopInfoStack::push(BasicBlock *Header, const llvm::DebugLoc &StartLoc,591 const llvm::DebugLoc &EndLoc) {592 Active.emplace_back(593 new LoopInfo(Header, StagedAttrs, StartLoc, EndLoc,594 Active.empty() ? nullptr : Active.back().get()));595 // Clear the attributes so nested loops do not inherit them.596 StagedAttrs.clear();597}598 599void LoopInfoStack::push(BasicBlock *Header, clang::ASTContext &Ctx,600 const clang::CodeGenOptions &CGOpts,601 ArrayRef<const clang::Attr *> Attrs,602 const llvm::DebugLoc &StartLoc,603 const llvm::DebugLoc &EndLoc, bool MustProgress) {604 // Identify loop hint attributes from Attrs.605 for (const auto *Attr : Attrs) {606 const LoopHintAttr *LH = dyn_cast<LoopHintAttr>(Attr);607 const OpenCLUnrollHintAttr *OpenCLHint =608 dyn_cast<OpenCLUnrollHintAttr>(Attr);609 const HLSLLoopHintAttr *HLSLLoopHint = dyn_cast<HLSLLoopHintAttr>(Attr);610 // Skip non loop hint attributes611 if (!LH && !OpenCLHint && !HLSLLoopHint) {612 continue;613 }614 615 LoopHintAttr::OptionType Option = LoopHintAttr::Unroll;616 LoopHintAttr::LoopHintState State = LoopHintAttr::Disable;617 unsigned ValueInt = 1;618 // Translate opencl_unroll_hint attribute argument to619 // equivalent LoopHintAttr enums.620 // OpenCL v2.0 s6.11.5:621 // 0 - enable unroll (no argument).622 // 1 - disable unroll.623 // other positive integer n - unroll by n.624 if (OpenCLHint) {625 ValueInt = OpenCLHint->getUnrollHint();626 if (ValueInt == 0) {627 State = LoopHintAttr::Enable;628 } else if (ValueInt != 1) {629 Option = LoopHintAttr::UnrollCount;630 State = LoopHintAttr::Numeric;631 }632 } else if (HLSLLoopHint) {633 ValueInt = HLSLLoopHint->getDirective();634 if (HLSLLoopHint->getSemanticSpelling() ==635 HLSLLoopHintAttr::Spelling::Microsoft_unroll) {636 if (ValueInt == 0)637 State = LoopHintAttr::Enable;638 if (ValueInt > 0) {639 Option = LoopHintAttr::UnrollCount;640 State = LoopHintAttr::Numeric;641 }642 }643 } else if (LH) {644 auto *ValueExpr = LH->getValue();645 if (ValueExpr) {646 llvm::APSInt ValueAPS = ValueExpr->EvaluateKnownConstInt(Ctx);647 ValueInt = ValueAPS.getSExtValue();648 }649 650 Option = LH->getOption();651 State = LH->getState();652 }653 switch (State) {654 case LoopHintAttr::Disable:655 switch (Option) {656 case LoopHintAttr::Vectorize:657 // Disable vectorization by specifying a width of 1.658 setVectorizeWidth(1);659 setVectorizeScalable(LoopAttributes::Unspecified);660 break;661 case LoopHintAttr::Interleave:662 // Disable interleaving by speciyfing a count of 1.663 setInterleaveCount(1);664 break;665 case LoopHintAttr::Unroll:666 setUnrollState(LoopAttributes::Disable);667 break;668 case LoopHintAttr::UnrollAndJam:669 setUnrollAndJamState(LoopAttributes::Disable);670 break;671 case LoopHintAttr::VectorizePredicate:672 setVectorizePredicateState(LoopAttributes::Disable);673 break;674 case LoopHintAttr::Distribute:675 setDistributeState(false);676 break;677 case LoopHintAttr::PipelineDisabled:678 setPipelineDisabled(true);679 break;680 case LoopHintAttr::UnrollCount:681 case LoopHintAttr::UnrollAndJamCount:682 case LoopHintAttr::VectorizeWidth:683 case LoopHintAttr::InterleaveCount:684 case LoopHintAttr::PipelineInitiationInterval:685 llvm_unreachable("Options cannot be disabled.");686 break;687 }688 break;689 case LoopHintAttr::Enable:690 switch (Option) {691 case LoopHintAttr::Vectorize:692 case LoopHintAttr::Interleave:693 setVectorizeEnable(true);694 break;695 case LoopHintAttr::Unroll:696 setUnrollState(LoopAttributes::Enable);697 break;698 case LoopHintAttr::UnrollAndJam:699 setUnrollAndJamState(LoopAttributes::Enable);700 break;701 case LoopHintAttr::VectorizePredicate:702 setVectorizePredicateState(LoopAttributes::Enable);703 break;704 case LoopHintAttr::Distribute:705 setDistributeState(true);706 break;707 case LoopHintAttr::UnrollCount:708 case LoopHintAttr::UnrollAndJamCount:709 case LoopHintAttr::VectorizeWidth:710 case LoopHintAttr::InterleaveCount:711 case LoopHintAttr::PipelineDisabled:712 case LoopHintAttr::PipelineInitiationInterval:713 llvm_unreachable("Options cannot enabled.");714 break;715 }716 break;717 case LoopHintAttr::AssumeSafety:718 switch (Option) {719 case LoopHintAttr::Vectorize:720 case LoopHintAttr::Interleave:721 // Apply "llvm.mem.parallel_loop_access" metadata to load/stores.722 setParallel(true);723 setVectorizeEnable(true);724 break;725 case LoopHintAttr::Unroll:726 case LoopHintAttr::UnrollAndJam:727 case LoopHintAttr::VectorizePredicate:728 case LoopHintAttr::UnrollCount:729 case LoopHintAttr::UnrollAndJamCount:730 case LoopHintAttr::VectorizeWidth:731 case LoopHintAttr::InterleaveCount:732 case LoopHintAttr::Distribute:733 case LoopHintAttr::PipelineDisabled:734 case LoopHintAttr::PipelineInitiationInterval:735 llvm_unreachable("Options cannot be used to assume mem safety.");736 break;737 }738 break;739 case LoopHintAttr::Full:740 switch (Option) {741 case LoopHintAttr::Unroll:742 setUnrollState(LoopAttributes::Full);743 break;744 case LoopHintAttr::UnrollAndJam:745 setUnrollAndJamState(LoopAttributes::Full);746 break;747 case LoopHintAttr::Vectorize:748 case LoopHintAttr::Interleave:749 case LoopHintAttr::UnrollCount:750 case LoopHintAttr::UnrollAndJamCount:751 case LoopHintAttr::VectorizeWidth:752 case LoopHintAttr::InterleaveCount:753 case LoopHintAttr::Distribute:754 case LoopHintAttr::PipelineDisabled:755 case LoopHintAttr::PipelineInitiationInterval:756 case LoopHintAttr::VectorizePredicate:757 llvm_unreachable("Options cannot be used with 'full' hint.");758 break;759 }760 break;761 case LoopHintAttr::FixedWidth:762 case LoopHintAttr::ScalableWidth:763 switch (Option) {764 case LoopHintAttr::VectorizeWidth:765 setVectorizeScalable(State == LoopHintAttr::ScalableWidth766 ? LoopAttributes::Enable767 : LoopAttributes::Disable);768 if (LH->getValue())769 setVectorizeWidth(ValueInt);770 break;771 default:772 llvm_unreachable("Options cannot be used with 'scalable' hint.");773 break;774 }775 break;776 case LoopHintAttr::Numeric:777 switch (Option) {778 case LoopHintAttr::InterleaveCount:779 setInterleaveCount(ValueInt);780 break;781 case LoopHintAttr::UnrollCount:782 setUnrollCount(ValueInt);783 break;784 case LoopHintAttr::UnrollAndJamCount:785 setUnrollAndJamCount(ValueInt);786 break;787 case LoopHintAttr::PipelineInitiationInterval:788 setPipelineInitiationInterval(ValueInt);789 break;790 case LoopHintAttr::Unroll:791 case LoopHintAttr::UnrollAndJam:792 case LoopHintAttr::VectorizePredicate:793 case LoopHintAttr::Vectorize:794 case LoopHintAttr::VectorizeWidth:795 case LoopHintAttr::Interleave:796 case LoopHintAttr::Distribute:797 case LoopHintAttr::PipelineDisabled:798 llvm_unreachable("Options cannot be assigned a value.");799 break;800 }801 break;802 }803 }804 805 // Identify loop attribute 'code_align' from Attrs.806 // For attribute code_align:807 // n - 'llvm.loop.align i32 n' metadata will be emitted.808 if (const auto *CodeAlign = getSpecificAttr<CodeAlignAttr>(Attrs)) {809 const auto *CE = cast<ConstantExpr>(CodeAlign->getAlignment());810 llvm::APSInt ArgVal = CE->getResultAsAPSInt();811 setCodeAlign(ArgVal.getSExtValue());812 }813 814 setMustProgress(MustProgress);815 816 if (CGOpts.OptimizationLevel > 0)817 // Disable unrolling for the loop, if unrolling is disabled (via818 // -fno-unroll-loops) and no pragmas override the decision.819 if (!CGOpts.UnrollLoops &&820 (StagedAttrs.UnrollEnable == LoopAttributes::Unspecified &&821 StagedAttrs.UnrollCount == 0))822 setUnrollState(LoopAttributes::Disable);823 824 /// Stage the attributes.825 push(Header, StartLoc, EndLoc);826}827 828void LoopInfoStack::pop() {829 assert(!Active.empty() && "No active loops to pop");830 Active.back()->finish();831 Active.pop_back();832}833 834void LoopInfoStack::InsertHelper(Instruction *I) const {835 if (I->mayReadOrWriteMemory()) {836 SmallVector<Metadata *, 4> AccessGroups;837 for (const auto &AL : Active) {838 // Here we assume that every loop that has an access group is parallel.839 if (MDNode *Group = AL->getAccessGroup())840 AccessGroups.push_back(Group);841 }842 MDNode *UnionMD = nullptr;843 if (AccessGroups.size() == 1)844 UnionMD = cast<MDNode>(AccessGroups[0]);845 else if (AccessGroups.size() >= 2)846 UnionMD = MDNode::get(I->getContext(), AccessGroups);847 I->setMetadata("llvm.access.group", UnionMD);848 }849 850 if (!hasInfo())851 return;852 853 const LoopInfo &L = getInfo();854 if (!L.getLoopID())855 return;856 857 if (I->isTerminator()) {858 for (BasicBlock *Succ : successors(I))859 if (Succ == L.getHeader()) {860 I->setMetadata(llvm::LLVMContext::MD_loop, L.getLoopID());861 break;862 }863 return;864 }865}866