//
// The LLVM Compiler Infrastructure
//
// This file is distributed under the University of Illinois Open Source
// License. See LICENSE.TXT for details.
//
//===----------------------------------------------------------------------===//
//
// This file contains a pass that performs optimization on SIMD instructions
// with high latency by splitting them into more efficient series of
// instructions.
//
// 1. Rewrite certain SIMD instructions with vector element due to their
// inefficiency on some targets.
//
// For example:
// fmla v0.4s, v1.4s, v2.s[1]
//
// Is rewritten into:
// dup v3.4s, v2.s[1]
// fmla v0.4s, v1.4s, v3.4s
//
// 2. Rewrite interleaved memory access instructions due to their
// inefficiency on some targets.
//
// For example:
// st2 {v0.4s, v1.4s}, addr
//
// Is rewritten into:
// zip1 v2.4s, v0.4s, v1.4s
// zip2 v3.4s, v0.4s, v1.4s
// stp q2, q3, addr
//
//===----------------------------------------------------------------------===//
#include "AArch64InstrInfo.h"
#include "llvm/ADT/SmallVector.h"
#include "llvm/ADT/Statistic.h"
#include "llvm/ADT/StringRef.h"
#include "llvm/CodeGen/MachineBasicBlock.h"
#include "llvm/CodeGen/MachineFunction.h"
#include "llvm/CodeGen/MachineFunctionPass.h"
#include "llvm/CodeGen/MachineInstr.h"
#include "llvm/CodeGen/MachineInstrBuilder.h"
#include "llvm/CodeGen/MachineOperand.h"
#include "llvm/CodeGen/MachineRegisterInfo.h"
#include "llvm/CodeGen/TargetInstrInfo.h"
#include "llvm/CodeGen/TargetSchedule.h"
#include "llvm/CodeGen/TargetSubtargetInfo.h"
#include "llvm/MC/MCInstrDesc.h"
#include "llvm/MC/MCSchedule.h"
#include "llvm/Pass.h"
#include <unordered_map>
using namespace llvm;
#define DEBUG_TYPE "aarch64-simdinstr-opt"
STATISTIC(NumModifiedInstr,
"Number of SIMD instructions modified");
#define AARCH64_VECTOR_BY_ELEMENT_OPT_NAME \
"AArch64 SIMD instructions optimization pass"
namespace {
struct AArch64SIMDInstrOpt : public MachineFunctionPass {
static char ID;
const TargetInstrInfo *TII;
MachineRegisterInfo *MRI;
TargetSchedModel SchedModel;
// The two maps below are used to cache decisions instead of recomputing:
// This is used to cache instruction replacement decisions within function
// units and across function units.
std::map<std::pair<unsigned, std::string>, bool> SIMDInstrTable;
// This is used to cache the decision of whether to leave the interleaved
// store instructions replacement pass early or not for a particular target.
std::unordered_map<std::string, bool> InterlEarlyExit;
typedef enum {
VectorElem,
Interleave
} Subpass;
// Instruction represented by OrigOpc is replaced by instructions in ReplOpc.
struct InstReplInfo {
unsigned OrigOpc;
std::vector<unsigned> ReplOpc;
const TargetRegisterClass RC;
};
#define RuleST2(OpcOrg, OpcR0, OpcR1, OpcR2, RC) \
{OpcOrg, {OpcR0, OpcR1, OpcR2}, RC}
#define RuleST4(OpcOrg, OpcR0, OpcR1, OpcR2, OpcR3, OpcR4, OpcR5, OpcR6, \
OpcR7, OpcR8, OpcR9, RC) \
{OpcOrg, \
{OpcR0, OpcR1, OpcR2, OpcR3, OpcR4, OpcR5, OpcR6, OpcR7, OpcR8, OpcR9}, RC}
// The Instruction Replacement Table:
std::vector<InstReplInfo> IRT = {
// ST2 instructions
RuleST2(AArch64::ST2Twov2d, AArch64::ZIP1v2i64, AArch64::ZIP2v2i64,
AArch64::STPQi, AArch64::FPR128RegClass),
RuleST2(AArch64::ST2Twov4s, AArch64::ZIP1v4i32, AArch64::ZIP2v4i32,
AArch64::STPQi, AArch64::FPR128RegClass),
RuleST2(AArch64::ST2Twov2s, AArch64::ZIP1v2i32, AArch64::ZIP2v2i32,
AArch64::STPDi, AArch64::FPR64RegClass),
RuleST2(AArch64::ST2Twov8h, AArch64::ZIP1v8i16, AArch64::ZIP2v8i16,
AArch64::STPQi, AArch64::FPR128RegClass),
RuleST2(AArch64::ST2Twov4h, AArch64::ZIP1v4i16, AArch64::ZIP2v4i16,
AArch64::STPDi, AArch64::FPR64RegClass),
RuleST2(AArch64::ST2Twov16b, AArch64::ZIP1v16i8, AArch64::ZIP2v16i8,
AArch64::STPQi, AArch64::FPR128RegClass),
RuleST2(AArch64::ST2Twov8b, AArch64::ZIP1v8i8, AArch64::ZIP2v8i8,
AArch64::STPDi, AArch64::FPR64RegClass),
// ST4 instructions
RuleST4(AArch64::ST4Fourv2d, AArch64::ZIP1v2i64, AArch64::ZIP2v2i64,
AArch64::ZIP1v2i64, AArch64::ZIP2v2i64, AArch64::ZIP1v2i64,
AArch64::ZIP2v2i64, AArch64::ZIP1v2i64, AArch64::ZIP2v2i64,
AArch64::STPQi, AArch64::STPQi, AArch64::FPR128RegClass),
RuleST4(AArch64::ST4Fourv4s, AArch64::ZIP1v4i32, AArch64::ZIP2v4i32,
AArch64::ZIP1v4i32, AArch64::ZIP2v4i32, AArch64::ZIP1v4i32,
AArch64::ZIP2v4i32, AArch64::ZIP1v4i32, AArch64::ZIP2v4i32,
AArch64::STPQi, AArch64::STPQi, AArch64::FPR128RegClass),
RuleST4(AArch64::ST4Fourv2s, AArch64::ZIP1v2i32, AArch64::ZIP2v2i32,
AArch64::ZIP1v2i32, AArch64::ZIP2v2i32, AArch64::ZIP1v2i32,
AArch64::ZIP2v2i32, AArch64::ZIP1v2i32, AArch64::ZIP2v2i32,
AArch64::STPDi, AArch64::STPDi, AArch64::FPR64RegClass),
RuleST4(AArch64::ST4Fourv8h, AArch64::ZIP1v8i16, AArch64::ZIP2v8i16,
AArch64::ZIP1v8i16, AArch64::ZIP2v8i16, AArch64::ZIP1v8i16,
AArch64::ZIP2v8i16, AArch64::ZIP1v8i16, AArch64::ZIP2v8i16,
AArch64::STPQi, AArch64::STPQi, AArch64::FPR128RegClass),
RuleST4(AArch64::ST4Fourv4h, AArch64::ZIP1v4i16, AArch64::ZIP2v4i16,
AArch64::ZIP1v4i16, AArch64::ZIP2v4i16, AArch64::ZIP1v4i16,
AArch64::ZIP2v4i16, AArch64::ZIP1v4i16, AArch64::ZIP2v4i16,
AArch64::STPDi, AArch64::STPDi, AArch64::FPR64RegClass),
RuleST4(AArch64::ST4Fourv16b, AArch64::ZIP1v16i8, AArch64::ZIP2v16i8,
AArch64::ZIP1v16i8, AArch64::ZIP2v16i8, AArch64::ZIP1v16i8,
AArch64::ZIP2v16i8, AArch64::ZIP1v16i8, AArch64::ZIP2v16i8,
AArch64::STPQi, AArch64::STPQi, AArch64::FPR128RegClass),
RuleST4(AArch64::ST4Fourv8b, AArch64::ZIP1v8i8, AArch64::ZIP2v8i8,
AArch64::ZIP1v8i8, AArch64::ZIP2v8i8, AArch64::ZIP1v8i8,
AArch64::ZIP2v8i8, AArch64::ZIP1v8i8, AArch64::ZIP2v8i8,
AArch64::STPDi, AArch64::STPDi, AArch64::FPR64RegClass)
};
// A costly instruction is replaced in this work by N efficient instructions
// The maximum of N is curently 10 and it is for ST4 case.
static const unsigned MaxNumRepl = 10;
AArch64SIMDInstrOpt() : MachineFunctionPass(ID) {
initializeAArch64SIMDInstrOptPass(*PassRegistry::getPassRegistry());
}
/// Based only on latency of instructions, determine if it is cost efficient
/// to replace the instruction InstDesc by the instructions stored in the
/// array InstDescRepl.
/// Return true if replacement is expected to be faster.
bool shouldReplaceInst(MachineFunction *MF, const MCInstrDesc *InstDesc,
SmallVectorImpl<const MCInstrDesc*> &ReplInstrMCID);
/// Determine if we need to exit the instruction replacement optimization
/// passes early. This makes sure that no compile time is spent in this pass
/// for targets with no need for any of these optimizations.
/// Return true if early exit of the pass is recommended.
bool shouldExitEarly(MachineFunction *MF, Subpass SP);
/// Check whether an equivalent DUP instruction has already been
/// created or not.
/// Return true when the DUP instruction already exists. In this case,
/// DestReg will point to the destination of the already created DUP.
bool reuseDUP(MachineInstr &MI, unsigned DupOpcode, unsigned SrcReg,
unsigned LaneNumber, unsigned *DestReg) const;
/// Certain SIMD instructions with vector element operand are not efficient.
/// Rewrite them into SIMD instructions with vector operands. This rewrite
/// is driven by the latency of the instructions.
/// Return true if the SIMD instruction is modified.
bool optimizeVectElement(MachineInstr &MI);
/// Process The REG_SEQUENCE instruction, and extract the source
/// operands of the ST2/4 instruction from it.
/// Example of such instructions.
/// %dest = REG_SEQUENCE %st2_src1, dsub0, %st2_src2, dsub1;
/// Return true when the instruction is processed successfully.
bool processSeqRegInst(MachineInstr *DefiningMI, unsigned* StReg,
unsigned* StRegKill, unsigned NumArg) const;
/// Load/Store Interleaving instructions are not always beneficial.
/// Replace them by ZIP instructionand classical load/store.
/// Return true if the SIMD instruction is modified.
bool optimizeLdStInterleave(MachineInstr &MI);
/// Return the number of useful source registers for this
/// instruction (2 for ST2 and 4 for ST4).
unsigned determineSrcReg(MachineInstr &MI) const;
bool runOnMachineFunction(MachineFunction &Fn) override;
StringRef getPassName() const override {
return AARCH64_VECTOR_BY_ELEMENT_OPT_NAME;
}
};
char AArch64SIMDInstrOpt::ID = 0;
} // end anonymous namespace
INITIALIZE_PASS(AArch64SIMDInstrOpt, "aarch64-simdinstr-opt",
AARCH64_VECTOR_BY_ELEMENT_OPT_NAME, false, false)
/// Based only on latency of instructions, determine if it is cost efficient
/// to replace the instruction InstDesc by the instructions stored in the
/// array InstDescRepl.
/// Return true if replacement is expected to be faster.
bool AArch64SIMDInstrOpt::
shouldReplaceInst(MachineFunction *MF, const MCInstrDesc *InstDesc,
SmallVectorImpl<const MCInstrDesc*> &InstDescRepl) {
// Check if replacement decision is already available in the cached table.
// if so, return it.
std::string Subtarget = SchedModel.getSubtargetInfo()->getCPU();
auto InstID = std::make_pair(InstDesc->getOpcode(), Subtarget);
if (SIMDInstrTable.find(InstID) != SIMDInstrTable.end())
return SIMDInstrTable[InstID];
unsigned SCIdx = InstDesc->getSchedClass();
const MCSchedClassDesc *SCDesc =
SchedModel.getMCSchedModel()->getSchedClassDesc(SCIdx);
// If a target does not define resources for the instructions
// of interest, then return false for no replacement.
const MCSchedClassDesc *SCDescRepl;
if (!SCDesc->isValid() || SCDesc->isVariant())
{
SIMDInstrTable[InstID] = false;
return false;
}
for (auto IDesc : InstDescRepl)
{
SCDescRepl = SchedModel.getMCSchedModel()->getSchedClassDesc(
IDesc->getSchedClass());
if (!SCDescRepl->isValid() || SCDescRepl->isVariant())
{
SIMDInstrTable[InstID] = false;
return false;
}
}
// Replacement cost.
unsigned ReplCost = 0;
for (auto IDesc :InstDescRepl)
ReplCost += SchedModel.computeInstrLatency(IDesc->getOpcode());
if (SchedModel.computeInstrLatency(InstDesc->getOpcode()) > ReplCost)
{
SIMDInstrTable[InstID] = true;
return true;
}
else
{
SIMDInstrTable[InstID] = false;
return false;
}
}
/// Determine if we need to exit this pass for a kind of instruction replacement
/// early. This makes sure that no compile time is spent in this pass for
/// targets with no need for any of these optimizations beyond performing this
/// check.
/// Return true if early exit of this pass for a kind of instruction
/// replacement is recommended for a target.
bool AArch64SIMDInstrOpt::shouldExitEarly(MachineFunction *MF, Subpass SP) {
const MCInstrDesc* OriginalMCID;
SmallVector<const MCInstrDesc*, MaxNumRepl> ReplInstrMCID;
switch (SP) {
// For this optimization, check by comparing the latency of a representative
// instruction to that of the replacement instructions.
// TODO: check for all concerned instructions.
case VectorElem:
OriginalMCID = &TII->get(AArch64::FMLAv4i32_indexed);
ReplInstrMCID.push_back(&TII->get(AArch64::DUPv4i32lane));
ReplInstrMCID.push_back(&TII->get(AArch64::FMLAv4f32));
if (shouldReplaceInst(MF, OriginalMCID, ReplInstrMCID))
return false;
break;
// For this optimization, check for all concerned instructions.
case Interleave:
std::string Subtarget = SchedModel.getSubtargetInfo()->getCPU();
if (InterlEarlyExit.find(Subtarget) != InterlEarlyExit.end())
return InterlEarlyExit[Subtarget];
for (auto &I : IRT) {
OriginalMCID = &TII->get(I.OrigOpc);
for (auto &Repl : I.ReplOpc)
ReplInstrMCID.push_back(&TII->get(Repl));
if (shouldReplaceInst(MF, OriginalMCID, ReplInstrMCID)) {
InterlEarlyExit[Subtarget] = false;
return false;
}
ReplInstrMCID.clear();
}
InterlEarlyExit[Subtarget] = true;
break;
}
return true;
}
/// Check whether an equivalent DUP instruction has already been
/// created or not.
/// Return true when the DUP instruction already exists. In this case,
/// DestReg will point to the destination of the already created DUP.
bool AArch64SIMDInstrOpt::reuseDUP(MachineInstr &MI, unsigned DupOpcode,
unsigned SrcReg, unsigned LaneNumber,
unsigned *DestReg) const {
for (MachineBasicBlock::iterator MII = MI, MIE = MI.getParent()->begin();
MII != MIE;) {
MII--;
MachineInstr *CurrentMI = &*MII;
if (CurrentMI->getOpcode() == DupOpcode &&
CurrentMI->getNumOperands() == 3 &&
CurrentMI->getOperand(1).getReg() == SrcReg &&
CurrentMI->getOperand(2).getImm() == LaneNumber) {
*DestReg = CurrentMI->getOperand(0).getReg();
return true;
}
}
return false;
}
/// Certain SIMD instructions with vector element operand are not efficient.
/// Rewrite them into SIMD instructions with vector operands. This rewrite
/// is driven by the latency of the instructions.
/// The instruction of concerns are for the time being FMLA, FMLS, FMUL,
/// and FMULX and hence they are hardcoded.
///
/// For example:
/// fmla v0.4s, v1.4s, v2.s[1]
///
/// Is rewritten into
/// dup v3.4s, v2.s[1] // DUP not necessary if redundant
/// fmla v0.4s, v1.4s, v3.4s
///
/// Return true if the SIMD instruction is modified.
bool AArch64SIMDInstrOpt::optimizeVectElement(MachineInstr &MI) {
const MCInstrDesc *MulMCID, *DupMCID;
const TargetRegisterClass *RC = &AArch64::FPR128RegClass;
switch (MI.getOpcode()) {
default:
return false;
// 4X32 instructions
case AArch64::FMLAv4i32_indexed:
DupMCID = &TII->get(AArch64::DUPv4i32lane);
MulMCID = &TII->get(AArch64::FMLAv4f32);
break;
case AArch64::FMLSv4i32_indexed:
DupMCID = &TII->get(AArch64::DUPv4i32lane);
MulMCID = &TII->get(AArch64::FMLSv4f32);
break;
case AArch64::FMULXv4i32_indexed:
DupMCID = &TII->get(AArch64::DUPv4i32lane);
MulMCID = &TII->get(AArch64::FMULXv4f32);
break;
case AArch64::FMULv4i32_indexed:
DupMCID = &TII->get(AArch64::DUPv4i32lane);
MulMCID = &TII->get(AArch64::FMULv4f32);
break;
// 2X64 instructions
case AArch64::FMLAv2i64_indexed:
DupMCID = &TII->get(AArch64::DUPv2i64lane);
MulMCID = &TII->get(AArch64::FMLAv2f64);
break;
case AArch64::FMLSv2i64_indexed:
DupMCID = &TII->get(AArch64::DUPv2i64lane);
MulMCID = &TII->get(AArch64::FMLSv2f64);
break;
case AArch64::FMULXv2i64_indexed:
DupMCID = &TII->get(AArch64::DUPv2i64lane);
MulMCID = &TII->get(AArch64::FMULXv2f64);
break;
case AArch64::FMULv2i64_indexed:
DupMCID = &TII->get(AArch64::DUPv2i64lane);
MulMCID = &TII->get(AArch64::FMULv2f64);
break;
// 2X32 instructions
case AArch64::FMLAv2i32_indexed:
RC = &AArch64::FPR64RegClass;
DupMCID = &TII->get(AArch64::DUPv2i32lane);
MulMCID = &TII->get(AArch64::FMLAv2f32);
break;
case AArch64::FMLSv2i32_indexed:
RC = &AArch64::FPR64RegClass;
DupMCID = &TII->get(AArch64::DUPv2i32lane);
MulMCID = &TII->get(AArch64::FMLSv2f32);
break;
case AArch64::FMULXv2i32_indexed:
RC = &AArch64::FPR64RegClass;
DupMCID = &TII->get(AArch64::DUPv2i32lane);
MulMCID = &TII->get(AArch64::FMULXv2f32);
break;
case AArch64::FMULv2i32_indexed:
RC = &AArch64::FPR64RegClass;
DupMCID = &TII->get(AArch64::DUPv2i32lane);
MulMCID = &TII->get(AArch64::FMULv2f32);
break;
}
SmallVector<const MCInstrDesc*, 2> ReplInstrMCID;
ReplInstrMCID.push_back(DupMCID);
ReplInstrMCID.push_back(MulMCID);
if (!shouldReplaceInst(MI.getParent()->getParent(), &TII->get(MI.getOpcode()),
ReplInstrMCID))
return false;
const DebugLoc &DL = MI.getDebugLoc();
MachineBasicBlock &MBB = *MI.getParent();
MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo();
// Get the operands of the current SIMD arithmetic instruction.
unsigned MulDest = MI.getOperand(0).getReg();
unsigned SrcReg0 = MI.getOperand(1).getReg();
unsigned Src0IsKill = getKillRegState(MI.getOperand(1).isKill());
unsigned SrcReg1 = MI.getOperand(2).getReg();
unsigned Src1IsKill = getKillRegState(MI.getOperand(2).isKill());
unsigned DupDest;
// Instructions of interest have either 4 or 5 operands.
if (MI.getNumOperands() == 5) {
unsigned SrcReg2 = MI.getOperand(3).getReg();
unsigned Src2IsKill = getKillRegState(MI.getOperand(3).isKill());
unsigned LaneNumber = MI.getOperand(4).getImm();
// Create a new DUP instruction. Note that if an equivalent DUP instruction
// has already been created before, then use that one instead of creating
// a new one.
if (!reuseDUP(MI, DupMCID->getOpcode(), SrcReg2, LaneNumber, &DupDest)) {
DupDest = MRI.createVirtualRegister(RC);
BuildMI(MBB, MI, DL, *DupMCID, DupDest)
.addReg(SrcReg2, Src2IsKill)
.addImm(LaneNumber);
}
BuildMI(MBB, MI, DL, *MulMCID, MulDest)
.addReg(SrcReg0, Src0IsKill)
.addReg(SrcReg1, Src1IsKill)
.addReg(DupDest, Src2IsKill);
} else if (MI.getNumOperands() == 4) {
unsigned LaneNumber = MI.getOperand(3).getImm();
if (!reuseDUP(MI, DupMCID->getOpcode(), SrcReg1, LaneNumber, &DupDest)) {
DupDest = MRI.createVirtualRegister(RC);
BuildMI(MBB, MI, DL, *DupMCID, DupDest)
.addReg(SrcReg1, Src1IsKill)
.addImm(LaneNumber);
}
BuildMI(MBB, MI, DL, *MulMCID, MulDest)
.addReg(SrcReg0, Src0IsKill)
.addReg(DupDest, Src1IsKill);
} else {
return false;
}
++NumModifiedInstr;
return true;
}
/// Load/Store Interleaving instructions are not always beneficial.
/// Replace them by ZIP instructions and classical load/store.
///
/// For example:
/// st2 {v0.4s, v1.4s}, addr
///
/// Is rewritten into:
/// zip1 v2.4s, v0.4s, v1.4s
/// zip2 v3.4s, v0.4s, v1.4s
/// stp q2, q3, addr
//
/// For example:
/// st4 {v0.4s, v1.4s, v2.4s, v3.4s}, addr
///
/// Is rewritten into:
/// zip1 v4.4s, v0.4s, v2.4s
/// zip2 v5.4s, v0.4s, v2.4s
/// zip1 v6.4s, v1.4s, v3.4s
/// zip2 v7.4s, v1.4s, v3.4s
/// zip1 v8.4s, v4.4s, v6.4s
/// zip2 v9.4s, v4.4s, v6.4s
/// zip1 v10.4s, v5.4s, v7.4s
/// zip2 v11.4s, v5.4s, v7.4s
/// stp q8, q9, addr
/// stp q10, q11, addr+32
///
/// Currently only instructions related to ST2 and ST4 are considered.
/// Other may be added later.
/// Return true if the SIMD instruction is modified.
bool AArch64SIMDInstrOpt::optimizeLdStInterleave(MachineInstr &MI) {
unsigned SeqReg, AddrReg;
unsigned StReg[4], StRegKill[4];
MachineInstr *DefiningMI;
const DebugLoc &DL = MI.getDebugLoc();
MachineBasicBlock &MBB = *MI.getParent();
SmallVector<unsigned, MaxNumRepl> ZipDest;
SmallVector<const MCInstrDesc*, MaxNumRepl> ReplInstrMCID;
// If current instruction matches any of the rewriting rules, then
// gather information about parameters of the new instructions.
bool Match = false;
for (auto &I : IRT) {
if (MI.getOpcode() == I.OrigOpc) {
SeqReg = MI.getOperand(0).getReg();
AddrReg = MI.getOperand(1).getReg();
DefiningMI = MRI->getUniqueVRegDef(SeqReg);
unsigned NumReg = determineSrcReg(MI);
if (!processSeqRegInst(DefiningMI, StReg, StRegKill, NumReg))
return false;
for (auto &Repl : I.ReplOpc) {
ReplInstrMCID.push_back(&TII->get(Repl));
// Generate destination registers but only for non-store instruction.
if (Repl != AArch64::STPQi && Repl != AArch64::STPDi)
ZipDest.push_back(MRI->createVirtualRegister(&I.RC));
}
Match = true;
break;
}
}
if (!Match)
return false;
// Determine if it is profitable to replace MI by the series of instructions
// represented in ReplInstrMCID.
if (!shouldReplaceInst(MI.getParent()->getParent(), &TII->get(MI.getOpcode()),
ReplInstrMCID))
return false;
// Generate the replacement instructions composed of ZIP1, ZIP2, and STP (at
// this point, the code generation is hardcoded and does not rely on the IRT
// table used above given that code generation for ST2 replacement is somewhat
// different than for ST4 replacement. We could have added more info into the
// table related to how we build new instructions but we may be adding more
// complexity with that).
switch (MI.getOpcode()) {
default:
return false;
case AArch64::ST2Twov16b:
case AArch64::ST2Twov8b:
case AArch64::ST2Twov8h:
case AArch64::ST2Twov4h:
case AArch64::ST2Twov4s:
case AArch64::ST2Twov2s:
case AArch64::ST2Twov2d:
// ZIP instructions
BuildMI(MBB, MI, DL, *ReplInstrMCID[0], ZipDest[0])
.addReg(StReg[0])
.addReg(StReg[1]);
BuildMI(MBB, MI, DL, *ReplInstrMCID[1], ZipDest[1])
.addReg(StReg[0], StRegKill[0])
.addReg(StReg[1], StRegKill[1]);
// STP instructions
BuildMI(MBB, MI, DL, *ReplInstrMCID[2])
.addReg(ZipDest[0])
.addReg(ZipDest[1])
.addReg(AddrReg)
.addImm(0);
break;
case AArch64::ST4Fourv16b:
case AArch64::ST4Fourv8b:
case AArch64::ST4Fourv8h:
case AArch64::ST4Fourv4h:
case AArch64::ST4Fourv4s:
case AArch64::ST4Fourv2s:
case AArch64::ST4Fourv2d:
// ZIP instructions
BuildMI(MBB, MI, DL, *ReplInstrMCID[0], ZipDest[0])
.addReg(StReg[0])
.addReg(StReg[2]);
BuildMI(MBB, MI, DL, *ReplInstrMCID[1], ZipDest[1])
.addReg(StReg[0], StRegKill[0])
.addReg(StReg[2], StRegKill[2]);
BuildMI(MBB, MI, DL, *ReplInstrMCID[2], ZipDest[2])
.addReg(StReg[1])
.addReg(StReg[3]);
BuildMI(MBB, MI, DL, *ReplInstrMCID[3], ZipDest[3])
.addReg(StReg[1], StRegKill[1])
.addReg(StReg[3], StRegKill[3]);
BuildMI(MBB, MI, DL, *ReplInstrMCID[4], ZipDest[4])
.addReg(ZipDest[0])
.addReg(ZipDest[2]);
BuildMI(MBB, MI, DL, *ReplInstrMCID[5], ZipDest[5])
.addReg(ZipDest[0])
.addReg(ZipDest[2]);
BuildMI(MBB, MI, DL, *ReplInstrMCID[6], ZipDest[6])
.addReg(ZipDest[1])
.addReg(ZipDest[3]);
BuildMI(MBB, MI, DL, *ReplInstrMCID[7], ZipDest[7])
.addReg(ZipDest[1])
.addReg(ZipDest[3]);
// stp instructions
BuildMI(MBB, MI, DL, *ReplInstrMCID[8])
.addReg(ZipDest[4])
.addReg(ZipDest[5])
.addReg(AddrReg)
.addImm(0);
BuildMI(MBB, MI, DL, *ReplInstrMCID[9])
.addReg(ZipDest[6])
.addReg(ZipDest[7])
.addReg(AddrReg)
.addImm(2);
break;
}
++NumModifiedInstr;
return true;
}
/// Process The REG_SEQUENCE instruction, and extract the source
/// operands of the ST2/4 instruction from it.
/// Example of such instruction.
/// %dest = REG_SEQUENCE %st2_src1, dsub0, %st2_src2, dsub1;
/// Return true when the instruction is processed successfully.
bool AArch64SIMDInstrOpt::processSeqRegInst(MachineInstr *DefiningMI,
unsigned* StReg, unsigned* StRegKill, unsigned NumArg) const {
assert (DefiningMI != NULL);
if (DefiningMI->getOpcode() != AArch64::REG_SEQUENCE)
return false;
for (unsigned i=0; i<NumArg; i++) {
StReg[i] = DefiningMI->getOperand(2*i+1).getReg();
StRegKill[i] = getKillRegState(DefiningMI->getOperand(2*i+1).isKill());
// Sanity check for the other arguments.
if (DefiningMI->getOperand(2*i+2).isImm()) {
switch (DefiningMI->getOperand(2*i+2).getImm()) {
default:
return false;
case AArch64::dsub0:
case AArch64::dsub1:
case AArch64::dsub2:
case AArch64::dsub3:
case AArch64::qsub0:
case AArch64::qsub1:
case AArch64::qsub2:
case AArch64::qsub3:
break;
}
}
else
return false;
}
return true;
}
/// Return the number of useful source registers for this instruction
/// (2 for ST2 and 4 for ST4).
unsigned AArch64SIMDInstrOpt::determineSrcReg(MachineInstr &MI) const {
switch (MI.getOpcode()) {
default:
llvm_unreachable("Unsupported instruction for this pass");
case AArch64::ST2Twov16b:
case AArch64::ST2Twov8b:
case AArch64::ST2Twov8h:
case AArch64::ST2Twov4h:
case AArch64::ST2Twov4s:
case AArch64::ST2Twov2s:
case AArch64::ST2Twov2d:
return 2;
case AArch64::ST4Fourv16b:
case AArch64::ST4Fourv8b:
case AArch64::ST4Fourv8h:
case AArch64::ST4Fourv4h:
case AArch64::ST4Fourv4s:
case AArch64::ST4Fourv2s:
case AArch64::ST4Fourv2d:
return 4;
}
}
bool AArch64SIMDInstrOpt::runOnMachineFunction(MachineFunction &MF) {
if (skipFunction(MF.getFunction()))
return false;
TII = MF.getSubtarget().getInstrInfo();
MRI = &MF.getRegInfo();
const TargetSubtargetInfo &ST = MF.getSubtarget();
const AArch64InstrInfo *AAII =
static_cast<const AArch64InstrInfo *>(ST.getInstrInfo());
if (!AAII)
return false;
SchedModel.init(&ST);
if (!SchedModel.hasInstrSchedModel())
return false;
bool Changed = false;
for (auto OptimizationKind : {VectorElem, Interleave}) {
if (!shouldExitEarly(&MF, OptimizationKind)) {
SmallVector<MachineInstr *, 8> RemoveMIs;
for (MachineBasicBlock &MBB : MF) {
for (MachineBasicBlock::iterator MII = MBB.begin(), MIE = MBB.end();
MII != MIE;) {
MachineInstr &MI = *MII;
bool InstRewrite;
if (OptimizationKind == VectorElem)
InstRewrite = optimizeVectElement(MI) ;
else
InstRewrite = optimizeLdStInterleave(MI);
if (InstRewrite) {
// Add MI to the list of instructions to be removed given that it
// has been replaced.
RemoveMIs.push_back(&MI);
Changed = true;
}
++MII;
}
}
for (MachineInstr *MI : RemoveMIs)
MI->eraseFromParent();
}
}
return Changed;
}
/// Returns an instance of the high cost ASIMD instruction replacement
/// optimization pass.
FunctionPass *llvm::createAArch64SIMDInstrOptPass() {
return new AArch64SIMDInstrOpt();
}