//===---- CGLoopInfo.cpp - LLVM CodeGen for loop metadata -*- C++ -*-------===// // // The LLVM Compiler Infrastructure // // This file is distributed under the University of Illinois Open Source // License. See LICENSE.TXT for details. // //===----------------------------------------------------------------------===// #include "CGLoopInfo.h" #include "clang/AST/ASTContext.h" #include "clang/AST/Attr.h" #include "clang/Sema/LoopHint.h" #include "llvm/IR/BasicBlock.h" #include "llvm/IR/Constants.h" #include "llvm/IR/InstrTypes.h" #include "llvm/IR/Instructions.h" #include "llvm/IR/Metadata.h" using namespace clang::CodeGen; using namespace llvm; static MDNode *createMetadata(LLVMContext &Ctx, const LoopAttributes &Attrs, llvm::DebugLoc Location) { if (!Attrs.IsParallel && Attrs.VectorizeWidth == 0 && Attrs.InterleaveCount == 0 && Attrs.UnrollCount == 0 && Attrs.VectorizeEnable == LoopAttributes::Unspecified && Attrs.UnrollEnable == LoopAttributes::Unspecified && Attrs.DistributeEnable == LoopAttributes::Unspecified && !Location) return nullptr; SmallVector<Metadata *, 4> Args; // Reserve operand 0 for loop id self reference. auto TempNode = MDNode::getTemporary(Ctx, None); Args.push_back(TempNode.get()); // If we have a valid debug location for the loop, add it. if (Location) Args.push_back(Location.getAsMDNode()); // Setting vectorize.width if (Attrs.VectorizeWidth > 0) { Metadata *Vals[] = {MDString::get(Ctx, "llvm.loop.vectorize.width"), ConstantAsMetadata::get(ConstantInt::get( Type::getInt32Ty(Ctx), Attrs.VectorizeWidth))}; Args.push_back(MDNode::get(Ctx, Vals)); } // Setting interleave.count if (Attrs.InterleaveCount > 0) { Metadata *Vals[] = {MDString::get(Ctx, "llvm.loop.interleave.count"), ConstantAsMetadata::get(ConstantInt::get( Type::getInt32Ty(Ctx), Attrs.InterleaveCount))}; Args.push_back(MDNode::get(Ctx, Vals)); } // Setting interleave.count if (Attrs.UnrollCount > 0) { Metadata *Vals[] = {MDString::get(Ctx, "llvm.loop.unroll.count"), ConstantAsMetadata::get(ConstantInt::get( Type::getInt32Ty(Ctx), Attrs.UnrollCount))}; Args.push_back(MDNode::get(Ctx, Vals)); } // Setting vectorize.enable if (Attrs.VectorizeEnable != LoopAttributes::Unspecified) { Metadata *Vals[] = {MDString::get(Ctx, "llvm.loop.vectorize.enable"), ConstantAsMetadata::get(ConstantInt::get( Type::getInt1Ty(Ctx), (Attrs.VectorizeEnable == LoopAttributes::Enable)))}; Args.push_back(MDNode::get(Ctx, Vals)); } // Setting unroll.full or unroll.disable if (Attrs.UnrollEnable != LoopAttributes::Unspecified) { std::string Name; if (Attrs.UnrollEnable == LoopAttributes::Enable) Name = "llvm.loop.unroll.enable"; else if (Attrs.UnrollEnable == LoopAttributes::Full) Name = "llvm.loop.unroll.full"; else Name = "llvm.loop.unroll.disable"; Metadata *Vals[] = {MDString::get(Ctx, Name)}; Args.push_back(MDNode::get(Ctx, Vals)); } if (Attrs.DistributeEnable != LoopAttributes::Unspecified) { Metadata *Vals[] = {MDString::get(Ctx, "llvm.loop.distribute.enable"), ConstantAsMetadata::get(ConstantInt::get( Type::getInt1Ty(Ctx), (Attrs.DistributeEnable == LoopAttributes::Enable)))}; Args.push_back(MDNode::get(Ctx, Vals)); } // Set the first operand to itself. MDNode *LoopID = MDNode::get(Ctx, Args); LoopID->replaceOperandWith(0, LoopID); return LoopID; } LoopAttributes::LoopAttributes(bool IsParallel) : IsParallel(IsParallel), VectorizeEnable(LoopAttributes::Unspecified), UnrollEnable(LoopAttributes::Unspecified), VectorizeWidth(0), InterleaveCount(0), UnrollCount(0), DistributeEnable(LoopAttributes::Unspecified) {} void LoopAttributes::clear() { IsParallel = false; VectorizeWidth = 0; InterleaveCount = 0; UnrollCount = 0; VectorizeEnable = LoopAttributes::Unspecified; UnrollEnable = LoopAttributes::Unspecified; } LoopInfo::LoopInfo(BasicBlock *Header, const LoopAttributes &Attrs, llvm::DebugLoc Location) : LoopID(nullptr), Header(Header), Attrs(Attrs) { LoopID = createMetadata(Header->getContext(), Attrs, Location); } void LoopInfoStack::push(BasicBlock *Header, llvm::DebugLoc Location) { Active.push_back(LoopInfo(Header, StagedAttrs, Location)); // Clear the attributes so nested loops do not inherit them. StagedAttrs.clear(); } void LoopInfoStack::push(BasicBlock *Header, clang::ASTContext &Ctx, ArrayRef<const clang::Attr *> Attrs, llvm::DebugLoc Location) { // Identify loop hint attributes from Attrs. for (const auto *Attr : Attrs) { const LoopHintAttr *LH = dyn_cast<LoopHintAttr>(Attr); const OpenCLUnrollHintAttr *OpenCLHint = dyn_cast<OpenCLUnrollHintAttr>(Attr); // Skip non loop hint attributes if (!LH && !OpenCLHint) { continue; } LoopHintAttr::OptionType Option = LoopHintAttr::Unroll; LoopHintAttr::LoopHintState State = LoopHintAttr::Disable; unsigned ValueInt = 1; // Translate opencl_unroll_hint attribute argument to // equivalent LoopHintAttr enums. // OpenCL v2.0 s6.11.5: // 0 - full unroll (no argument). // 1 - disable unroll. // other positive integer n - unroll by n. if (OpenCLHint) { ValueInt = OpenCLHint->getUnrollHint(); if (ValueInt == 0) { State = LoopHintAttr::Full; } else if (ValueInt != 1) { Option = LoopHintAttr::UnrollCount; State = LoopHintAttr::Numeric; } } else if (LH) { auto *ValueExpr = LH->getValue(); if (ValueExpr) { llvm::APSInt ValueAPS = ValueExpr->EvaluateKnownConstInt(Ctx); ValueInt = ValueAPS.getSExtValue(); } Option = LH->getOption(); State = LH->getState(); } switch (State) { case LoopHintAttr::Disable: switch (Option) { case LoopHintAttr::Vectorize: // Disable vectorization by specifying a width of 1. setVectorizeWidth(1); break; case LoopHintAttr::Interleave: // Disable interleaving by speciyfing a count of 1. setInterleaveCount(1); break; case LoopHintAttr::Unroll: setUnrollState(LoopAttributes::Disable); break; case LoopHintAttr::Distribute: setDistributeState(false); break; case LoopHintAttr::UnrollCount: case LoopHintAttr::VectorizeWidth: case LoopHintAttr::InterleaveCount: llvm_unreachable("Options cannot be disabled."); break; } break; case LoopHintAttr::Enable: switch (Option) { case LoopHintAttr::Vectorize: case LoopHintAttr::Interleave: setVectorizeEnable(true); break; case LoopHintAttr::Unroll: setUnrollState(LoopAttributes::Enable); break; case LoopHintAttr::Distribute: setDistributeState(true); break; case LoopHintAttr::UnrollCount: case LoopHintAttr::VectorizeWidth: case LoopHintAttr::InterleaveCount: llvm_unreachable("Options cannot enabled."); break; } break; case LoopHintAttr::AssumeSafety: switch (Option) { case LoopHintAttr::Vectorize: case LoopHintAttr::Interleave: // Apply "llvm.mem.parallel_loop_access" metadata to load/stores. setParallel(true); setVectorizeEnable(true); break; case LoopHintAttr::Unroll: case LoopHintAttr::UnrollCount: case LoopHintAttr::VectorizeWidth: case LoopHintAttr::InterleaveCount: case LoopHintAttr::Distribute: llvm_unreachable("Options cannot be used to assume mem safety."); break; } break; case LoopHintAttr::Full: switch (Option) { case LoopHintAttr::Unroll: setUnrollState(LoopAttributes::Full); break; case LoopHintAttr::Vectorize: case LoopHintAttr::Interleave: case LoopHintAttr::UnrollCount: case LoopHintAttr::VectorizeWidth: case LoopHintAttr::InterleaveCount: case LoopHintAttr::Distribute: llvm_unreachable("Options cannot be used with 'full' hint."); break; } break; case LoopHintAttr::Numeric: switch (Option) { case LoopHintAttr::VectorizeWidth: setVectorizeWidth(ValueInt); break; case LoopHintAttr::InterleaveCount: setInterleaveCount(ValueInt); break; case LoopHintAttr::UnrollCount: setUnrollCount(ValueInt); break; case LoopHintAttr::Unroll: case LoopHintAttr::Vectorize: case LoopHintAttr::Interleave: case LoopHintAttr::Distribute: llvm_unreachable("Options cannot be assigned a value."); break; } break; } } /// Stage the attributes. push(Header, Location); } void LoopInfoStack::pop() { assert(!Active.empty() && "No active loops to pop"); Active.pop_back(); } void LoopInfoStack::InsertHelper(Instruction *I) const { if (!hasInfo()) return; const LoopInfo &L = getInfo(); if (!L.getLoopID()) return; if (TerminatorInst *TI = dyn_cast<TerminatorInst>(I)) { for (unsigned i = 0, ie = TI->getNumSuccessors(); i < ie; ++i) if (TI->getSuccessor(i) == L.getHeader()) { TI->setMetadata(llvm::LLVMContext::MD_loop, L.getLoopID()); break; } return; } if (L.getAttributes().IsParallel && I->mayReadOrWriteMemory()) I->setMetadata("llvm.mem.parallel_loop_access", L.getLoopID()); }