//===- SPIRVInstructionSelector.cpp ------------------------------*- C++ -*-==// // // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. // See https://llvm.org/LICENSE.txt for license information. // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // // This file implements the targeting of the InstructionSelector class for // SPIRV. // TODO: This should be generated by TableGen. // //===----------------------------------------------------------------------===// #include "MCTargetDesc/SPIRVBaseInfo.h" #include "MCTargetDesc/SPIRVMCTargetDesc.h" #include "SPIRV.h" #include "SPIRVGlobalRegistry.h" #include "SPIRVInstrInfo.h" #include "SPIRVRegisterBankInfo.h" #include "SPIRVRegisterInfo.h" #include "SPIRVTargetMachine.h" #include "SPIRVUtils.h" #include "llvm/ADT/APFloat.h" #include "llvm/CodeGen/GlobalISel/GIMatchTableExecutorImpl.h" #include "llvm/CodeGen/GlobalISel/GenericMachineInstrs.h" #include "llvm/CodeGen/GlobalISel/InstructionSelector.h" #include "llvm/CodeGen/MachineInstrBuilder.h" #include "llvm/CodeGen/MachineModuleInfoImpls.h" #include "llvm/CodeGen/MachineRegisterInfo.h" #include "llvm/CodeGen/TargetOpcodes.h" #include "llvm/IR/IntrinsicsSPIRV.h" #include "llvm/Support/Debug.h" namespace llvm { class SPIRVMachineModuleInfo : public MachineModuleInfoImpl { public: SyncScope::ID Work_ItemSSID; SyncScope::ID WorkGroupSSID; SyncScope::ID DeviceSSID; SyncScope::ID AllSVMDevicesSSID; SyncScope::ID SubGroupSSID; SPIRVMachineModuleInfo(const MachineModuleInfo &MMI) { LLVMContext &CTX = MMI.getModule()->getContext(); Work_ItemSSID = CTX.getOrInsertSyncScopeID("work_item"); WorkGroupSSID = CTX.getOrInsertSyncScopeID("workgroup"); DeviceSSID = CTX.getOrInsertSyncScopeID("device"); AllSVMDevicesSSID = CTX.getOrInsertSyncScopeID("all_svm_devices"); SubGroupSSID = CTX.getOrInsertSyncScopeID("sub_group"); } }; } // end namespace llvm #define DEBUG_TYPE "spirv-isel" using namespace llvm; namespace CL = SPIRV::OpenCLExtInst; namespace GL = SPIRV::GLSLExtInst; using ExtInstList = std::vector>; namespace { #define GET_GLOBALISEL_PREDICATE_BITSET #include "SPIRVGenGlobalISel.inc" #undef GET_GLOBALISEL_PREDICATE_BITSET class SPIRVInstructionSelector : public InstructionSelector { const SPIRVSubtarget &STI; const SPIRVInstrInfo &TII; const SPIRVRegisterInfo &TRI; const RegisterBankInfo &RBI; SPIRVGlobalRegistry &GR; MachineRegisterInfo *MRI; SPIRVMachineModuleInfo *MMI = nullptr; /// We need to keep track of the number we give to anonymous global values to /// generate the same name every time when this is needed. mutable DenseMap UnnamedGlobalIDs; public: SPIRVInstructionSelector(const SPIRVTargetMachine &TM, const SPIRVSubtarget &ST, const RegisterBankInfo &RBI); void setupMF(MachineFunction &MF, GISelKnownBits *KB, CodeGenCoverage *CoverageInfo, ProfileSummaryInfo *PSI, BlockFrequencyInfo *BFI) override; // Common selection code. Instruction-specific selection occurs in spvSelect. bool select(MachineInstr &I) override; static const char *getName() { return DEBUG_TYPE; } #define GET_GLOBALISEL_PREDICATES_DECL #include "SPIRVGenGlobalISel.inc" #undef GET_GLOBALISEL_PREDICATES_DECL #define GET_GLOBALISEL_TEMPORARIES_DECL #include "SPIRVGenGlobalISel.inc" #undef GET_GLOBALISEL_TEMPORARIES_DECL private: // tblgen-erated 'select' implementation, used as the initial selector for // the patterns that don't require complex C++. bool selectImpl(MachineInstr &I, CodeGenCoverage &CoverageInfo) const; // All instruction-specific selection that didn't happen in "select()". // Is basically a large Switch/Case delegating to all other select method. bool spvSelect(Register ResVReg, const SPIRVType *ResType, MachineInstr &I) const; bool selectGlobalValue(Register ResVReg, MachineInstr &I, const MachineInstr *Init = nullptr) const; bool selectUnOpWithSrc(Register ResVReg, const SPIRVType *ResType, MachineInstr &I, Register SrcReg, unsigned Opcode) const; bool selectUnOp(Register ResVReg, const SPIRVType *ResType, MachineInstr &I, unsigned Opcode) const; bool selectBitcast(Register ResVReg, const SPIRVType *ResType, MachineInstr &I) const; bool selectLoad(Register ResVReg, const SPIRVType *ResType, MachineInstr &I) const; bool selectStore(MachineInstr &I) const; bool selectStackSave(Register ResVReg, const SPIRVType *ResType, MachineInstr &I) const; bool selectStackRestore(MachineInstr &I) const; bool selectMemOperation(Register ResVReg, MachineInstr &I) const; bool selectAtomicRMW(Register ResVReg, const SPIRVType *ResType, MachineInstr &I, unsigned NewOpcode, unsigned NegateOpcode = 0) const; bool selectAtomicCmpXchg(Register ResVReg, const SPIRVType *ResType, MachineInstr &I) const; bool selectFence(MachineInstr &I) const; bool selectAddrSpaceCast(Register ResVReg, const SPIRVType *ResType, MachineInstr &I) const; bool selectAnyOrAll(Register ResVReg, const SPIRVType *ResType, MachineInstr &I, unsigned OpType) const; bool selectAll(Register ResVReg, const SPIRVType *ResType, MachineInstr &I) const; bool selectAny(Register ResVReg, const SPIRVType *ResType, MachineInstr &I) const; bool selectBitreverse(Register ResVReg, const SPIRVType *ResType, MachineInstr &I) const; bool selectConstVector(Register ResVReg, const SPIRVType *ResType, MachineInstr &I) const; bool selectSplatVector(Register ResVReg, const SPIRVType *ResType, MachineInstr &I) const; bool selectCmp(Register ResVReg, const SPIRVType *ResType, unsigned comparisonOpcode, MachineInstr &I) const; bool selectICmp(Register ResVReg, const SPIRVType *ResType, MachineInstr &I) const; bool selectFCmp(Register ResVReg, const SPIRVType *ResType, MachineInstr &I) const; bool selectFmix(Register ResVReg, const SPIRVType *ResType, MachineInstr &I) const; bool selectRsqrt(Register ResVReg, const SPIRVType *ResType, MachineInstr &I) const; void renderImm32(MachineInstrBuilder &MIB, const MachineInstr &I, int OpIdx) const; void renderFImm32(MachineInstrBuilder &MIB, const MachineInstr &I, int OpIdx) const; bool selectConst(Register ResVReg, const SPIRVType *ResType, const APInt &Imm, MachineInstr &I) const; bool selectSelect(Register ResVReg, const SPIRVType *ResType, MachineInstr &I, bool IsSigned) const; bool selectIToF(Register ResVReg, const SPIRVType *ResType, MachineInstr &I, bool IsSigned, unsigned Opcode) const; bool selectExt(Register ResVReg, const SPIRVType *ResType, MachineInstr &I, bool IsSigned) const; bool selectTrunc(Register ResVReg, const SPIRVType *ResType, MachineInstr &I) const; bool selectIntToBool(Register IntReg, Register ResVReg, MachineInstr &I, const SPIRVType *intTy, const SPIRVType *boolTy) const; bool selectOpUndef(Register ResVReg, const SPIRVType *ResType, MachineInstr &I) const; bool selectFreeze(Register ResVReg, const SPIRVType *ResType, MachineInstr &I) const; bool selectIntrinsic(Register ResVReg, const SPIRVType *ResType, MachineInstr &I) const; bool selectExtractVal(Register ResVReg, const SPIRVType *ResType, MachineInstr &I) const; bool selectInsertVal(Register ResVReg, const SPIRVType *ResType, MachineInstr &I) const; bool selectExtractElt(Register ResVReg, const SPIRVType *ResType, MachineInstr &I) const; bool selectInsertElt(Register ResVReg, const SPIRVType *ResType, MachineInstr &I) const; bool selectGEP(Register ResVReg, const SPIRVType *ResType, MachineInstr &I) const; bool selectFrameIndex(Register ResVReg, const SPIRVType *ResType, MachineInstr &I) const; bool selectAllocaArray(Register ResVReg, const SPIRVType *ResType, MachineInstr &I) const; bool selectBranch(MachineInstr &I) const; bool selectBranchCond(MachineInstr &I) const; bool selectPhi(Register ResVReg, const SPIRVType *ResType, MachineInstr &I) const; bool selectExtInst(Register ResVReg, const SPIRVType *ResType, MachineInstr &I, CL::OpenCLExtInst CLInst) const; bool selectExtInst(Register ResVReg, const SPIRVType *ResType, MachineInstr &I, CL::OpenCLExtInst CLInst, GL::GLSLExtInst GLInst) const; bool selectExtInst(Register ResVReg, const SPIRVType *ResType, MachineInstr &I, const ExtInstList &ExtInsts) const; bool selectLog10(Register ResVReg, const SPIRVType *ResType, MachineInstr &I) const; bool selectSpvThreadId(Register ResVReg, const SPIRVType *ResType, MachineInstr &I) const; bool selectUnmergeValues(MachineInstr &I) const; Register buildI32Constant(uint32_t Val, MachineInstr &I, const SPIRVType *ResType = nullptr) const; Register buildZerosVal(const SPIRVType *ResType, MachineInstr &I) const; Register buildZerosValF(const SPIRVType *ResType, MachineInstr &I) const; Register buildOnesVal(bool AllOnes, const SPIRVType *ResType, MachineInstr &I) const; bool wrapIntoSpecConstantOp(MachineInstr &I, SmallVector &CompositeArgs) const; }; } // end anonymous namespace #define GET_GLOBALISEL_IMPL #include "SPIRVGenGlobalISel.inc" #undef GET_GLOBALISEL_IMPL SPIRVInstructionSelector::SPIRVInstructionSelector(const SPIRVTargetMachine &TM, const SPIRVSubtarget &ST, const RegisterBankInfo &RBI) : InstructionSelector(), STI(ST), TII(*ST.getInstrInfo()), TRI(*ST.getRegisterInfo()), RBI(RBI), GR(*ST.getSPIRVGlobalRegistry()), #define GET_GLOBALISEL_PREDICATES_INIT #include "SPIRVGenGlobalISel.inc" #undef GET_GLOBALISEL_PREDICATES_INIT #define GET_GLOBALISEL_TEMPORARIES_INIT #include "SPIRVGenGlobalISel.inc" #undef GET_GLOBALISEL_TEMPORARIES_INIT { } void SPIRVInstructionSelector::setupMF(MachineFunction &MF, GISelKnownBits *KB, CodeGenCoverage *CoverageInfo, ProfileSummaryInfo *PSI, BlockFrequencyInfo *BFI) { MMI = &MF.getMMI().getObjFileInfo(); MRI = &MF.getRegInfo(); GR.setCurrentFunc(MF); InstructionSelector::setupMF(MF, KB, CoverageInfo, PSI, BFI); } static bool isImm(const MachineOperand &MO, MachineRegisterInfo *MRI); // Defined in SPIRVLegalizerInfo.cpp. extern bool isTypeFoldingSupported(unsigned Opcode); bool SPIRVInstructionSelector::select(MachineInstr &I) { assert(I.getParent() && "Instruction should be in a basic block!"); assert(I.getParent()->getParent() && "Instruction should be in a function!"); Register Opcode = I.getOpcode(); // If it's not a GMIR instruction, we've selected it already. if (!isPreISelGenericOpcode(Opcode)) { if (Opcode == SPIRV::ASSIGN_TYPE) { // These pseudos aren't needed any more. Register DstReg = I.getOperand(0).getReg(); Register SrcReg = I.getOperand(1).getReg(); auto *Def = MRI->getVRegDef(SrcReg); if (isTypeFoldingSupported(Def->getOpcode())) { if (MRI->getType(DstReg).isPointer()) MRI->setType(DstReg, LLT::scalar(32)); bool Res = selectImpl(I, *CoverageInfo); assert(Res || Def->getOpcode() == TargetOpcode::G_CONSTANT); if (Res) return Res; } MRI->replaceRegWith(SrcReg, DstReg); I.removeFromParent(); return true; } else if (I.getNumDefs() == 1) { // Make all vregs 32 bits (for SPIR-V IDs). MRI->setType(I.getOperand(0).getReg(), LLT::scalar(32)); } return constrainSelectedInstRegOperands(I, TII, TRI, RBI); } if (I.getNumOperands() != I.getNumExplicitOperands()) { LLVM_DEBUG(errs() << "Generic instr has unexpected implicit operands\n"); return false; } // Common code for getting return reg+type, and removing selected instr // from parent occurs here. Instr-specific selection happens in spvSelect(). bool HasDefs = I.getNumDefs() > 0; Register ResVReg = HasDefs ? I.getOperand(0).getReg() : Register(0); SPIRVType *ResType = HasDefs ? GR.getSPIRVTypeForVReg(ResVReg) : nullptr; assert(!HasDefs || ResType || I.getOpcode() == TargetOpcode::G_GLOBAL_VALUE); if (spvSelect(ResVReg, ResType, I)) { if (HasDefs) // Make all vregs 32 bits (for SPIR-V IDs). for (unsigned i = 0; i < I.getNumDefs(); ++i) MRI->setType(I.getOperand(i).getReg(), LLT::scalar(32)); I.removeFromParent(); return true; } return false; } bool SPIRVInstructionSelector::spvSelect(Register ResVReg, const SPIRVType *ResType, MachineInstr &I) const { const unsigned Opcode = I.getOpcode(); if (isTypeFoldingSupported(Opcode) && Opcode != TargetOpcode::G_CONSTANT) return selectImpl(I, *CoverageInfo); switch (Opcode) { case TargetOpcode::G_CONSTANT: return selectConst(ResVReg, ResType, I.getOperand(1).getCImm()->getValue(), I); case TargetOpcode::G_GLOBAL_VALUE: return selectGlobalValue(ResVReg, I); case TargetOpcode::G_IMPLICIT_DEF: return selectOpUndef(ResVReg, ResType, I); case TargetOpcode::G_FREEZE: return selectFreeze(ResVReg, ResType, I); case TargetOpcode::G_INTRINSIC: case TargetOpcode::G_INTRINSIC_W_SIDE_EFFECTS: case TargetOpcode::G_INTRINSIC_CONVERGENT_W_SIDE_EFFECTS: return selectIntrinsic(ResVReg, ResType, I); case TargetOpcode::G_BITREVERSE: return selectBitreverse(ResVReg, ResType, I); case TargetOpcode::G_BUILD_VECTOR: return selectConstVector(ResVReg, ResType, I); case TargetOpcode::G_SPLAT_VECTOR: return selectSplatVector(ResVReg, ResType, I); case TargetOpcode::G_SHUFFLE_VECTOR: { MachineBasicBlock &BB = *I.getParent(); auto MIB = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpVectorShuffle)) .addDef(ResVReg) .addUse(GR.getSPIRVTypeID(ResType)) .addUse(I.getOperand(1).getReg()) .addUse(I.getOperand(2).getReg()); for (auto V : I.getOperand(3).getShuffleMask()) MIB.addImm(V); return MIB.constrainAllUses(TII, TRI, RBI); } case TargetOpcode::G_MEMMOVE: case TargetOpcode::G_MEMCPY: case TargetOpcode::G_MEMSET: return selectMemOperation(ResVReg, I); case TargetOpcode::G_ICMP: return selectICmp(ResVReg, ResType, I); case TargetOpcode::G_FCMP: return selectFCmp(ResVReg, ResType, I); case TargetOpcode::G_FRAME_INDEX: return selectFrameIndex(ResVReg, ResType, I); case TargetOpcode::G_LOAD: return selectLoad(ResVReg, ResType, I); case TargetOpcode::G_STORE: return selectStore(I); case TargetOpcode::G_BR: return selectBranch(I); case TargetOpcode::G_BRCOND: return selectBranchCond(I); case TargetOpcode::G_PHI: return selectPhi(ResVReg, ResType, I); case TargetOpcode::G_FPTOSI: return selectUnOp(ResVReg, ResType, I, SPIRV::OpConvertFToS); case TargetOpcode::G_FPTOUI: return selectUnOp(ResVReg, ResType, I, SPIRV::OpConvertFToU); case TargetOpcode::G_SITOFP: return selectIToF(ResVReg, ResType, I, true, SPIRV::OpConvertSToF); case TargetOpcode::G_UITOFP: return selectIToF(ResVReg, ResType, I, false, SPIRV::OpConvertUToF); case TargetOpcode::G_CTPOP: return selectUnOp(ResVReg, ResType, I, SPIRV::OpBitCount); case TargetOpcode::G_SMIN: return selectExtInst(ResVReg, ResType, I, CL::s_min, GL::SMin); case TargetOpcode::G_UMIN: return selectExtInst(ResVReg, ResType, I, CL::u_min, GL::UMin); case TargetOpcode::G_SMAX: return selectExtInst(ResVReg, ResType, I, CL::s_max, GL::SMax); case TargetOpcode::G_UMAX: return selectExtInst(ResVReg, ResType, I, CL::u_max, GL::UMax); case TargetOpcode::G_FMA: return selectExtInst(ResVReg, ResType, I, CL::fma, GL::Fma); case TargetOpcode::G_FPOW: return selectExtInst(ResVReg, ResType, I, CL::pow, GL::Pow); case TargetOpcode::G_FPOWI: return selectExtInst(ResVReg, ResType, I, CL::pown); case TargetOpcode::G_FEXP: return selectExtInst(ResVReg, ResType, I, CL::exp, GL::Exp); case TargetOpcode::G_FEXP2: return selectExtInst(ResVReg, ResType, I, CL::exp2, GL::Exp2); case TargetOpcode::G_FLOG: return selectExtInst(ResVReg, ResType, I, CL::log, GL::Log); case TargetOpcode::G_FLOG2: return selectExtInst(ResVReg, ResType, I, CL::log2, GL::Log2); case TargetOpcode::G_FLOG10: return selectLog10(ResVReg, ResType, I); case TargetOpcode::G_FABS: return selectExtInst(ResVReg, ResType, I, CL::fabs, GL::FAbs); case TargetOpcode::G_ABS: return selectExtInst(ResVReg, ResType, I, CL::s_abs, GL::SAbs); case TargetOpcode::G_FMINNUM: case TargetOpcode::G_FMINIMUM: return selectExtInst(ResVReg, ResType, I, CL::fmin, GL::NMin); case TargetOpcode::G_FMAXNUM: case TargetOpcode::G_FMAXIMUM: return selectExtInst(ResVReg, ResType, I, CL::fmax, GL::NMax); case TargetOpcode::G_FCOPYSIGN: return selectExtInst(ResVReg, ResType, I, CL::copysign); case TargetOpcode::G_FCEIL: return selectExtInst(ResVReg, ResType, I, CL::ceil, GL::Ceil); case TargetOpcode::G_FFLOOR: return selectExtInst(ResVReg, ResType, I, CL::floor, GL::Floor); case TargetOpcode::G_FCOS: return selectExtInst(ResVReg, ResType, I, CL::cos, GL::Cos); case TargetOpcode::G_FSIN: return selectExtInst(ResVReg, ResType, I, CL::sin, GL::Sin); case TargetOpcode::G_FTAN: return selectExtInst(ResVReg, ResType, I, CL::tan, GL::Tan); case TargetOpcode::G_FACOS: return selectExtInst(ResVReg, ResType, I, CL::acos, GL::Acos); case TargetOpcode::G_FASIN: return selectExtInst(ResVReg, ResType, I, CL::asin, GL::Asin); case TargetOpcode::G_FATAN: return selectExtInst(ResVReg, ResType, I, CL::atan, GL::Atan); case TargetOpcode::G_FCOSH: return selectExtInst(ResVReg, ResType, I, CL::cosh, GL::Cosh); case TargetOpcode::G_FSINH: return selectExtInst(ResVReg, ResType, I, CL::sinh, GL::Sinh); case TargetOpcode::G_FTANH: return selectExtInst(ResVReg, ResType, I, CL::tanh, GL::Tanh); case TargetOpcode::G_FSQRT: return selectExtInst(ResVReg, ResType, I, CL::sqrt, GL::Sqrt); case TargetOpcode::G_CTTZ: case TargetOpcode::G_CTTZ_ZERO_UNDEF: return selectExtInst(ResVReg, ResType, I, CL::ctz); case TargetOpcode::G_CTLZ: case TargetOpcode::G_CTLZ_ZERO_UNDEF: return selectExtInst(ResVReg, ResType, I, CL::clz); case TargetOpcode::G_INTRINSIC_ROUND: return selectExtInst(ResVReg, ResType, I, CL::round, GL::Round); case TargetOpcode::G_INTRINSIC_ROUNDEVEN: return selectExtInst(ResVReg, ResType, I, CL::rint, GL::RoundEven); case TargetOpcode::G_INTRINSIC_TRUNC: return selectExtInst(ResVReg, ResType, I, CL::trunc, GL::Trunc); case TargetOpcode::G_FRINT: case TargetOpcode::G_FNEARBYINT: return selectExtInst(ResVReg, ResType, I, CL::rint, GL::RoundEven); case TargetOpcode::G_SMULH: return selectExtInst(ResVReg, ResType, I, CL::s_mul_hi); case TargetOpcode::G_UMULH: return selectExtInst(ResVReg, ResType, I, CL::u_mul_hi); case TargetOpcode::G_SADDSAT: return selectExtInst(ResVReg, ResType, I, CL::s_add_sat); case TargetOpcode::G_UADDSAT: return selectExtInst(ResVReg, ResType, I, CL::u_add_sat); case TargetOpcode::G_SSUBSAT: return selectExtInst(ResVReg, ResType, I, CL::s_sub_sat); case TargetOpcode::G_USUBSAT: return selectExtInst(ResVReg, ResType, I, CL::u_sub_sat); case TargetOpcode::G_SEXT: return selectExt(ResVReg, ResType, I, true); case TargetOpcode::G_ANYEXT: case TargetOpcode::G_ZEXT: return selectExt(ResVReg, ResType, I, false); case TargetOpcode::G_TRUNC: return selectTrunc(ResVReg, ResType, I); case TargetOpcode::G_FPTRUNC: case TargetOpcode::G_FPEXT: return selectUnOp(ResVReg, ResType, I, SPIRV::OpFConvert); case TargetOpcode::G_PTRTOINT: return selectUnOp(ResVReg, ResType, I, SPIRV::OpConvertPtrToU); case TargetOpcode::G_INTTOPTR: return selectUnOp(ResVReg, ResType, I, SPIRV::OpConvertUToPtr); case TargetOpcode::G_BITCAST: return selectBitcast(ResVReg, ResType, I); case TargetOpcode::G_ADDRSPACE_CAST: return selectAddrSpaceCast(ResVReg, ResType, I); case TargetOpcode::G_PTR_ADD: { // Currently, we get G_PTR_ADD only as a result of translating // global variables, initialized with constant expressions like GV + Const // (see test opencl/basic/progvar_prog_scope_init.ll). // TODO: extend the handler once we have other cases. assert(I.getOperand(1).isReg() && I.getOperand(2).isReg()); Register GV = I.getOperand(1).getReg(); MachineRegisterInfo::def_instr_iterator II = MRI->def_instr_begin(GV); (void)II; assert(((*II).getOpcode() == TargetOpcode::G_GLOBAL_VALUE || (*II).getOpcode() == TargetOpcode::COPY || (*II).getOpcode() == SPIRV::OpVariable) && isImm(I.getOperand(2), MRI)); Register Idx = buildZerosVal(GR.getOrCreateSPIRVIntegerType(32, I, TII), I); MachineBasicBlock &BB = *I.getParent(); auto MIB = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpSpecConstantOp)) .addDef(ResVReg) .addUse(GR.getSPIRVTypeID(ResType)) .addImm(static_cast( SPIRV::Opcode::InBoundsPtrAccessChain)) .addUse(GV) .addUse(Idx) .addUse(I.getOperand(2).getReg()); return MIB.constrainAllUses(TII, TRI, RBI); } case TargetOpcode::G_ATOMICRMW_OR: return selectAtomicRMW(ResVReg, ResType, I, SPIRV::OpAtomicOr); case TargetOpcode::G_ATOMICRMW_ADD: return selectAtomicRMW(ResVReg, ResType, I, SPIRV::OpAtomicIAdd); case TargetOpcode::G_ATOMICRMW_AND: return selectAtomicRMW(ResVReg, ResType, I, SPIRV::OpAtomicAnd); case TargetOpcode::G_ATOMICRMW_MAX: return selectAtomicRMW(ResVReg, ResType, I, SPIRV::OpAtomicSMax); case TargetOpcode::G_ATOMICRMW_MIN: return selectAtomicRMW(ResVReg, ResType, I, SPIRV::OpAtomicSMin); case TargetOpcode::G_ATOMICRMW_SUB: return selectAtomicRMW(ResVReg, ResType, I, SPIRV::OpAtomicISub); case TargetOpcode::G_ATOMICRMW_XOR: return selectAtomicRMW(ResVReg, ResType, I, SPIRV::OpAtomicXor); case TargetOpcode::G_ATOMICRMW_UMAX: return selectAtomicRMW(ResVReg, ResType, I, SPIRV::OpAtomicUMax); case TargetOpcode::G_ATOMICRMW_UMIN: return selectAtomicRMW(ResVReg, ResType, I, SPIRV::OpAtomicUMin); case TargetOpcode::G_ATOMICRMW_XCHG: return selectAtomicRMW(ResVReg, ResType, I, SPIRV::OpAtomicExchange); case TargetOpcode::G_ATOMIC_CMPXCHG: return selectAtomicCmpXchg(ResVReg, ResType, I); case TargetOpcode::G_ATOMICRMW_FADD: return selectAtomicRMW(ResVReg, ResType, I, SPIRV::OpAtomicFAddEXT); case TargetOpcode::G_ATOMICRMW_FSUB: // Translate G_ATOMICRMW_FSUB to OpAtomicFAddEXT with negative value operand return selectAtomicRMW(ResVReg, ResType, I, SPIRV::OpAtomicFAddEXT, SPIRV::OpFNegate); case TargetOpcode::G_ATOMICRMW_FMIN: return selectAtomicRMW(ResVReg, ResType, I, SPIRV::OpAtomicFMinEXT); case TargetOpcode::G_ATOMICRMW_FMAX: return selectAtomicRMW(ResVReg, ResType, I, SPIRV::OpAtomicFMaxEXT); case TargetOpcode::G_FENCE: return selectFence(I); case TargetOpcode::G_STACKSAVE: return selectStackSave(ResVReg, ResType, I); case TargetOpcode::G_STACKRESTORE: return selectStackRestore(I); case TargetOpcode::G_UNMERGE_VALUES: return selectUnmergeValues(I); default: return false; } } bool SPIRVInstructionSelector::selectExtInst(Register ResVReg, const SPIRVType *ResType, MachineInstr &I, CL::OpenCLExtInst CLInst) const { return selectExtInst(ResVReg, ResType, I, {{SPIRV::InstructionSet::OpenCL_std, CLInst}}); } bool SPIRVInstructionSelector::selectExtInst(Register ResVReg, const SPIRVType *ResType, MachineInstr &I, CL::OpenCLExtInst CLInst, GL::GLSLExtInst GLInst) const { ExtInstList ExtInsts = {{SPIRV::InstructionSet::OpenCL_std, CLInst}, {SPIRV::InstructionSet::GLSL_std_450, GLInst}}; return selectExtInst(ResVReg, ResType, I, ExtInsts); } bool SPIRVInstructionSelector::selectExtInst(Register ResVReg, const SPIRVType *ResType, MachineInstr &I, const ExtInstList &Insts) const { for (const auto &Ex : Insts) { SPIRV::InstructionSet::InstructionSet Set = Ex.first; uint32_t Opcode = Ex.second; if (STI.canUseExtInstSet(Set)) { MachineBasicBlock &BB = *I.getParent(); auto MIB = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpExtInst)) .addDef(ResVReg) .addUse(GR.getSPIRVTypeID(ResType)) .addImm(static_cast(Set)) .addImm(Opcode); const unsigned NumOps = I.getNumOperands(); for (unsigned i = 1; i < NumOps; ++i) MIB.add(I.getOperand(i)); return MIB.constrainAllUses(TII, TRI, RBI); } } return false; } bool SPIRVInstructionSelector::selectUnOpWithSrc(Register ResVReg, const SPIRVType *ResType, MachineInstr &I, Register SrcReg, unsigned Opcode) const { return BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(Opcode)) .addDef(ResVReg) .addUse(GR.getSPIRVTypeID(ResType)) .addUse(SrcReg) .constrainAllUses(TII, TRI, RBI); } bool SPIRVInstructionSelector::selectUnOp(Register ResVReg, const SPIRVType *ResType, MachineInstr &I, unsigned Opcode) const { if (STI.isOpenCLEnv() && I.getOperand(1).isReg()) { Register SrcReg = I.getOperand(1).getReg(); bool IsGV = false; for (MachineRegisterInfo::def_instr_iterator DefIt = MRI->def_instr_begin(SrcReg); DefIt != MRI->def_instr_end(); DefIt = std::next(DefIt)) { if ((*DefIt).getOpcode() == TargetOpcode::G_GLOBAL_VALUE) { IsGV = true; break; } } if (IsGV) { uint32_t SpecOpcode = 0; switch (Opcode) { case SPIRV::OpConvertPtrToU: SpecOpcode = static_cast(SPIRV::Opcode::ConvertPtrToU); break; case SPIRV::OpConvertUToPtr: SpecOpcode = static_cast(SPIRV::Opcode::ConvertUToPtr); break; } if (SpecOpcode) return BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(SPIRV::OpSpecConstantOp)) .addDef(ResVReg) .addUse(GR.getSPIRVTypeID(ResType)) .addImm(SpecOpcode) .addUse(SrcReg) .constrainAllUses(TII, TRI, RBI); } } return selectUnOpWithSrc(ResVReg, ResType, I, I.getOperand(1).getReg(), Opcode); } bool SPIRVInstructionSelector::selectBitcast(Register ResVReg, const SPIRVType *ResType, MachineInstr &I) const { Register OpReg = I.getOperand(1).getReg(); SPIRVType *OpType = OpReg.isValid() ? GR.getSPIRVTypeForVReg(OpReg) : nullptr; if (!GR.isBitcastCompatible(ResType, OpType)) report_fatal_error("incompatible result and operand types in a bitcast"); return selectUnOp(ResVReg, ResType, I, SPIRV::OpBitcast); } static SPIRV::Scope::Scope getScope(SyncScope::ID Ord, SPIRVMachineModuleInfo *MMI) { if (Ord == SyncScope::SingleThread || Ord == MMI->Work_ItemSSID) return SPIRV::Scope::Invocation; else if (Ord == SyncScope::System || Ord == MMI->DeviceSSID) return SPIRV::Scope::Device; else if (Ord == MMI->WorkGroupSSID) return SPIRV::Scope::Workgroup; else if (Ord == MMI->AllSVMDevicesSSID) return SPIRV::Scope::CrossDevice; else if (Ord == MMI->SubGroupSSID) return SPIRV::Scope::Subgroup; else // OpenCL approach is: "The functions that do not have memory_scope argument // have the same semantics as the corresponding functions with the // memory_scope argument set to memory_scope_device." See ref.: // // https://registry.khronos.org/OpenCL/specs/3.0-unified/html/OpenCL_C.html#atomic-functions // In our case if the scope is unknown, assuming that SPIR-V code is to be // consumed in an OpenCL environment, we use the same approach and set the // scope to memory_scope_device. return SPIRV::Scope::Device; } static void addMemoryOperands(MachineMemOperand *MemOp, MachineInstrBuilder &MIB) { uint32_t SpvMemOp = static_cast(SPIRV::MemoryOperand::None); if (MemOp->isVolatile()) SpvMemOp |= static_cast(SPIRV::MemoryOperand::Volatile); if (MemOp->isNonTemporal()) SpvMemOp |= static_cast(SPIRV::MemoryOperand::Nontemporal); if (MemOp->getAlign().value()) SpvMemOp |= static_cast(SPIRV::MemoryOperand::Aligned); if (SpvMemOp != static_cast(SPIRV::MemoryOperand::None)) { MIB.addImm(SpvMemOp); if (SpvMemOp & static_cast(SPIRV::MemoryOperand::Aligned)) MIB.addImm(MemOp->getAlign().value()); } } static void addMemoryOperands(uint64_t Flags, MachineInstrBuilder &MIB) { uint32_t SpvMemOp = static_cast(SPIRV::MemoryOperand::None); if (Flags & MachineMemOperand::Flags::MOVolatile) SpvMemOp |= static_cast(SPIRV::MemoryOperand::Volatile); if (Flags & MachineMemOperand::Flags::MONonTemporal) SpvMemOp |= static_cast(SPIRV::MemoryOperand::Nontemporal); if (SpvMemOp != static_cast(SPIRV::MemoryOperand::None)) MIB.addImm(SpvMemOp); } bool SPIRVInstructionSelector::selectLoad(Register ResVReg, const SPIRVType *ResType, MachineInstr &I) const { unsigned OpOffset = isa(I) ? 1 : 0; Register Ptr = I.getOperand(1 + OpOffset).getReg(); auto MIB = BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(SPIRV::OpLoad)) .addDef(ResVReg) .addUse(GR.getSPIRVTypeID(ResType)) .addUse(Ptr); if (!I.getNumMemOperands()) { assert(I.getOpcode() == TargetOpcode::G_INTRINSIC_W_SIDE_EFFECTS || I.getOpcode() == TargetOpcode::G_INTRINSIC_CONVERGENT_W_SIDE_EFFECTS); addMemoryOperands(I.getOperand(2 + OpOffset).getImm(), MIB); } else { addMemoryOperands(*I.memoperands_begin(), MIB); } return MIB.constrainAllUses(TII, TRI, RBI); } bool SPIRVInstructionSelector::selectStore(MachineInstr &I) const { unsigned OpOffset = isa(I) ? 1 : 0; Register StoreVal = I.getOperand(0 + OpOffset).getReg(); Register Ptr = I.getOperand(1 + OpOffset).getReg(); MachineBasicBlock &BB = *I.getParent(); auto MIB = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpStore)) .addUse(Ptr) .addUse(StoreVal); if (!I.getNumMemOperands()) { assert(I.getOpcode() == TargetOpcode::G_INTRINSIC_W_SIDE_EFFECTS || I.getOpcode() == TargetOpcode::G_INTRINSIC_CONVERGENT_W_SIDE_EFFECTS); addMemoryOperands(I.getOperand(2 + OpOffset).getImm(), MIB); } else { addMemoryOperands(*I.memoperands_begin(), MIB); } return MIB.constrainAllUses(TII, TRI, RBI); } bool SPIRVInstructionSelector::selectStackSave(Register ResVReg, const SPIRVType *ResType, MachineInstr &I) const { if (!STI.canUseExtension(SPIRV::Extension::SPV_INTEL_variable_length_array)) report_fatal_error( "llvm.stacksave intrinsic: this instruction requires the following " "SPIR-V extension: SPV_INTEL_variable_length_array", false); MachineBasicBlock &BB = *I.getParent(); return BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpSaveMemoryINTEL)) .addDef(ResVReg) .addUse(GR.getSPIRVTypeID(ResType)) .constrainAllUses(TII, TRI, RBI); } bool SPIRVInstructionSelector::selectStackRestore(MachineInstr &I) const { if (!STI.canUseExtension(SPIRV::Extension::SPV_INTEL_variable_length_array)) report_fatal_error( "llvm.stackrestore intrinsic: this instruction requires the following " "SPIR-V extension: SPV_INTEL_variable_length_array", false); if (!I.getOperand(0).isReg()) return false; MachineBasicBlock &BB = *I.getParent(); return BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpRestoreMemoryINTEL)) .addUse(I.getOperand(0).getReg()) .constrainAllUses(TII, TRI, RBI); } bool SPIRVInstructionSelector::selectMemOperation(Register ResVReg, MachineInstr &I) const { MachineBasicBlock &BB = *I.getParent(); Register SrcReg = I.getOperand(1).getReg(); if (I.getOpcode() == TargetOpcode::G_MEMSET) { assert(I.getOperand(1).isReg() && I.getOperand(2).isReg()); unsigned Val = getIConstVal(I.getOperand(1).getReg(), MRI); unsigned Num = getIConstVal(I.getOperand(2).getReg(), MRI); SPIRVType *ValTy = GR.getOrCreateSPIRVIntegerType(8, I, TII); SPIRVType *ArrTy = GR.getOrCreateSPIRVArrayType(ValTy, Num, I, TII); Register Const = GR.getOrCreateConstIntArray(Val, Num, I, ArrTy, TII); SPIRVType *VarTy = GR.getOrCreateSPIRVPointerType( ArrTy, I, TII, SPIRV::StorageClass::UniformConstant); // TODO: check if we have such GV, add init, use buildGlobalVariable. Function &CurFunction = GR.CurMF->getFunction(); Type *LLVMArrTy = ArrayType::get(IntegerType::get(CurFunction.getContext(), 8), Num); // Module takes ownership of the global var. GlobalVariable *GV = new GlobalVariable(*CurFunction.getParent(), LLVMArrTy, true, GlobalValue::InternalLinkage, Constant::getNullValue(LLVMArrTy)); Register VarReg = MRI->createGenericVirtualRegister(LLT::scalar(32)); GR.add(GV, GR.CurMF, VarReg); buildOpDecorate(VarReg, I, TII, SPIRV::Decoration::Constant, {}); BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(SPIRV::OpVariable)) .addDef(VarReg) .addUse(GR.getSPIRVTypeID(VarTy)) .addImm(SPIRV::StorageClass::UniformConstant) .addUse(Const) .constrainAllUses(TII, TRI, RBI); SPIRVType *SourceTy = GR.getOrCreateSPIRVPointerType( ValTy, I, TII, SPIRV::StorageClass::UniformConstant); SrcReg = MRI->createGenericVirtualRegister(LLT::scalar(32)); selectUnOpWithSrc(SrcReg, SourceTy, I, VarReg, SPIRV::OpBitcast); } auto MIB = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpCopyMemorySized)) .addUse(I.getOperand(0).getReg()) .addUse(SrcReg) .addUse(I.getOperand(2).getReg()); if (I.getNumMemOperands()) addMemoryOperands(*I.memoperands_begin(), MIB); bool Result = MIB.constrainAllUses(TII, TRI, RBI); if (ResVReg.isValid() && ResVReg != MIB->getOperand(0).getReg()) BuildMI(BB, I, I.getDebugLoc(), TII.get(TargetOpcode::COPY), ResVReg) .addUse(MIB->getOperand(0).getReg()); return Result; } bool SPIRVInstructionSelector::selectAtomicRMW(Register ResVReg, const SPIRVType *ResType, MachineInstr &I, unsigned NewOpcode, unsigned NegateOpcode) const { assert(I.hasOneMemOperand()); const MachineMemOperand *MemOp = *I.memoperands_begin(); uint32_t Scope = static_cast(getScope(MemOp->getSyncScopeID(), MMI)); Register ScopeReg = buildI32Constant(Scope, I); Register Ptr = I.getOperand(1).getReg(); // TODO: Changed as it's implemented in the translator. See test/atomicrmw.ll // auto ScSem = // getMemSemanticsForStorageClass(GR.getPointerStorageClass(Ptr)); AtomicOrdering AO = MemOp->getSuccessOrdering(); uint32_t MemSem = static_cast(getMemSemantics(AO)); Register MemSemReg = buildI32Constant(MemSem /*| ScSem*/, I); bool Result = false; Register ValueReg = I.getOperand(2).getReg(); if (NegateOpcode != 0) { // Translation with negative value operand is requested Register TmpReg = MRI->createVirtualRegister(&SPIRV::IDRegClass); Result |= selectUnOpWithSrc(TmpReg, ResType, I, ValueReg, NegateOpcode); ValueReg = TmpReg; } Result |= BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(NewOpcode)) .addDef(ResVReg) .addUse(GR.getSPIRVTypeID(ResType)) .addUse(Ptr) .addUse(ScopeReg) .addUse(MemSemReg) .addUse(ValueReg) .constrainAllUses(TII, TRI, RBI); return Result; } bool SPIRVInstructionSelector::selectUnmergeValues(MachineInstr &I) const { unsigned ArgI = I.getNumOperands() - 1; Register SrcReg = I.getOperand(ArgI).isReg() ? I.getOperand(ArgI).getReg() : Register(0); SPIRVType *DefType = SrcReg.isValid() ? GR.getSPIRVTypeForVReg(SrcReg) : nullptr; if (!DefType || DefType->getOpcode() != SPIRV::OpTypeVector) report_fatal_error( "cannot select G_UNMERGE_VALUES with a non-vector argument"); SPIRVType *ScalarType = GR.getSPIRVTypeForVReg(DefType->getOperand(1).getReg()); MachineBasicBlock &BB = *I.getParent(); bool Res = false; for (unsigned i = 0; i < I.getNumDefs(); ++i) { Register ResVReg = I.getOperand(i).getReg(); SPIRVType *ResType = GR.getSPIRVTypeForVReg(ResVReg); if (!ResType) { // There was no "assign type" actions, let's fix this now ResType = ScalarType; MRI->setRegClass(ResVReg, &SPIRV::IDRegClass); MRI->setType(ResVReg, LLT::scalar(GR.getScalarOrVectorBitWidth(ResType))); GR.assignSPIRVTypeToVReg(ResType, ResVReg, *GR.CurMF); } auto MIB = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpCompositeExtract)) .addDef(ResVReg) .addUse(GR.getSPIRVTypeID(ResType)) .addUse(SrcReg) .addImm(static_cast(i)); Res |= MIB.constrainAllUses(TII, TRI, RBI); } return Res; } bool SPIRVInstructionSelector::selectFence(MachineInstr &I) const { AtomicOrdering AO = AtomicOrdering(I.getOperand(0).getImm()); uint32_t MemSem = static_cast(getMemSemantics(AO)); Register MemSemReg = buildI32Constant(MemSem, I); SyncScope::ID Ord = SyncScope::ID(I.getOperand(1).getImm()); uint32_t Scope = static_cast(getScope(Ord, MMI)); Register ScopeReg = buildI32Constant(Scope, I); MachineBasicBlock &BB = *I.getParent(); return BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpMemoryBarrier)) .addUse(ScopeReg) .addUse(MemSemReg) .constrainAllUses(TII, TRI, RBI); } bool SPIRVInstructionSelector::selectAtomicCmpXchg(Register ResVReg, const SPIRVType *ResType, MachineInstr &I) const { Register ScopeReg; Register MemSemEqReg; Register MemSemNeqReg; Register Ptr = I.getOperand(2).getReg(); if (!isa(I)) { assert(I.hasOneMemOperand()); const MachineMemOperand *MemOp = *I.memoperands_begin(); unsigned Scope = static_cast(getScope(MemOp->getSyncScopeID(), MMI)); ScopeReg = buildI32Constant(Scope, I); unsigned ScSem = static_cast( getMemSemanticsForStorageClass(GR.getPointerStorageClass(Ptr))); AtomicOrdering AO = MemOp->getSuccessOrdering(); unsigned MemSemEq = static_cast(getMemSemantics(AO)) | ScSem; MemSemEqReg = buildI32Constant(MemSemEq, I); AtomicOrdering FO = MemOp->getFailureOrdering(); unsigned MemSemNeq = static_cast(getMemSemantics(FO)) | ScSem; MemSemNeqReg = MemSemEq == MemSemNeq ? MemSemEqReg : buildI32Constant(MemSemNeq, I); } else { ScopeReg = I.getOperand(5).getReg(); MemSemEqReg = I.getOperand(6).getReg(); MemSemNeqReg = I.getOperand(7).getReg(); } Register Cmp = I.getOperand(3).getReg(); Register Val = I.getOperand(4).getReg(); SPIRVType *SpvValTy = GR.getSPIRVTypeForVReg(Val); Register ACmpRes = MRI->createVirtualRegister(&SPIRV::IDRegClass); const DebugLoc &DL = I.getDebugLoc(); bool Result = BuildMI(*I.getParent(), I, DL, TII.get(SPIRV::OpAtomicCompareExchange)) .addDef(ACmpRes) .addUse(GR.getSPIRVTypeID(SpvValTy)) .addUse(Ptr) .addUse(ScopeReg) .addUse(MemSemEqReg) .addUse(MemSemNeqReg) .addUse(Val) .addUse(Cmp) .constrainAllUses(TII, TRI, RBI); Register CmpSuccReg = MRI->createVirtualRegister(&SPIRV::IDRegClass); SPIRVType *BoolTy = GR.getOrCreateSPIRVBoolType(I, TII); Result |= BuildMI(*I.getParent(), I, DL, TII.get(SPIRV::OpIEqual)) .addDef(CmpSuccReg) .addUse(GR.getSPIRVTypeID(BoolTy)) .addUse(ACmpRes) .addUse(Cmp) .constrainAllUses(TII, TRI, RBI); Register TmpReg = MRI->createVirtualRegister(&SPIRV::IDRegClass); Result |= BuildMI(*I.getParent(), I, DL, TII.get(SPIRV::OpCompositeInsert)) .addDef(TmpReg) .addUse(GR.getSPIRVTypeID(ResType)) .addUse(ACmpRes) .addUse(GR.getOrCreateUndef(I, ResType, TII)) .addImm(0) .constrainAllUses(TII, TRI, RBI); Result |= BuildMI(*I.getParent(), I, DL, TII.get(SPIRV::OpCompositeInsert)) .addDef(ResVReg) .addUse(GR.getSPIRVTypeID(ResType)) .addUse(CmpSuccReg) .addUse(TmpReg) .addImm(1) .constrainAllUses(TII, TRI, RBI); return Result; } static bool isGenericCastablePtr(SPIRV::StorageClass::StorageClass SC) { switch (SC) { case SPIRV::StorageClass::Workgroup: case SPIRV::StorageClass::CrossWorkgroup: case SPIRV::StorageClass::Function: return true; default: return false; } } static bool isUSMStorageClass(SPIRV::StorageClass::StorageClass SC) { switch (SC) { case SPIRV::StorageClass::DeviceOnlyINTEL: case SPIRV::StorageClass::HostOnlyINTEL: return true; default: return false; } } // In SPIR-V address space casting can only happen to and from the Generic // storage class. We can also only cast Workgroup, CrossWorkgroup, or Function // pointers to and from Generic pointers. As such, we can convert e.g. from // Workgroup to Function by going via a Generic pointer as an intermediary. All // other combinations can only be done by a bitcast, and are probably not safe. bool SPIRVInstructionSelector::selectAddrSpaceCast(Register ResVReg, const SPIRVType *ResType, MachineInstr &I) const { // If the AddrSpaceCast user is single and in OpConstantComposite or // OpVariable, we should select OpSpecConstantOp. auto UIs = MRI->use_instructions(ResVReg); if (!UIs.empty() && ++UIs.begin() == UIs.end() && (UIs.begin()->getOpcode() == SPIRV::OpConstantComposite || UIs.begin()->getOpcode() == SPIRV::OpVariable || isSpvIntrinsic(*UIs.begin(), Intrinsic::spv_init_global))) { Register NewReg = I.getOperand(1).getReg(); MachineBasicBlock &BB = *I.getParent(); SPIRVType *SpvBaseTy = GR.getOrCreateSPIRVIntegerType(8, I, TII); ResType = GR.getOrCreateSPIRVPointerType(SpvBaseTy, I, TII, SPIRV::StorageClass::Generic); bool Result = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpSpecConstantOp)) .addDef(ResVReg) .addUse(GR.getSPIRVTypeID(ResType)) .addImm(static_cast(SPIRV::Opcode::PtrCastToGeneric)) .addUse(NewReg) .constrainAllUses(TII, TRI, RBI); return Result; } Register SrcPtr = I.getOperand(1).getReg(); SPIRVType *SrcPtrTy = GR.getSPIRVTypeForVReg(SrcPtr); SPIRV::StorageClass::StorageClass SrcSC = GR.getPointerStorageClass(SrcPtr); SPIRV::StorageClass::StorageClass DstSC = GR.getPointerStorageClass(ResVReg); // don't generate a cast between identical storage classes if (SrcSC == DstSC) return BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(TargetOpcode::COPY)) .addDef(ResVReg) .addUse(SrcPtr) .constrainAllUses(TII, TRI, RBI); // Casting from an eligible pointer to Generic. if (DstSC == SPIRV::StorageClass::Generic && isGenericCastablePtr(SrcSC)) return selectUnOp(ResVReg, ResType, I, SPIRV::OpPtrCastToGeneric); // Casting from Generic to an eligible pointer. if (SrcSC == SPIRV::StorageClass::Generic && isGenericCastablePtr(DstSC)) return selectUnOp(ResVReg, ResType, I, SPIRV::OpGenericCastToPtr); // Casting between 2 eligible pointers using Generic as an intermediary. if (isGenericCastablePtr(SrcSC) && isGenericCastablePtr(DstSC)) { Register Tmp = MRI->createVirtualRegister(&SPIRV::IDRegClass); SPIRVType *GenericPtrTy = GR.getOrCreateSPIRVPointerType( GR.getPointeeType(SrcPtrTy), I, TII, SPIRV::StorageClass::Generic); MachineBasicBlock &BB = *I.getParent(); const DebugLoc &DL = I.getDebugLoc(); bool Success = BuildMI(BB, I, DL, TII.get(SPIRV::OpPtrCastToGeneric)) .addDef(Tmp) .addUse(GR.getSPIRVTypeID(GenericPtrTy)) .addUse(SrcPtr) .constrainAllUses(TII, TRI, RBI); return Success && BuildMI(BB, I, DL, TII.get(SPIRV::OpGenericCastToPtr)) .addDef(ResVReg) .addUse(GR.getSPIRVTypeID(ResType)) .addUse(Tmp) .constrainAllUses(TII, TRI, RBI); } // Check if instructions from the SPV_INTEL_usm_storage_classes extension may // be applied if (isUSMStorageClass(SrcSC) && DstSC == SPIRV::StorageClass::CrossWorkgroup) return selectUnOp(ResVReg, ResType, I, SPIRV::OpPtrCastToCrossWorkgroupINTEL); if (SrcSC == SPIRV::StorageClass::CrossWorkgroup && isUSMStorageClass(DstSC)) return selectUnOp(ResVReg, ResType, I, SPIRV::OpCrossWorkgroupCastToPtrINTEL); if (isUSMStorageClass(SrcSC) && DstSC == SPIRV::StorageClass::Generic) return selectUnOp(ResVReg, ResType, I, SPIRV::OpPtrCastToGeneric); if (SrcSC == SPIRV::StorageClass::Generic && isUSMStorageClass(DstSC)) return selectUnOp(ResVReg, ResType, I, SPIRV::OpGenericCastToPtr); // Bitcast for pointers requires that the address spaces must match return false; } static unsigned getFCmpOpcode(unsigned PredNum) { auto Pred = static_cast(PredNum); switch (Pred) { case CmpInst::FCMP_OEQ: return SPIRV::OpFOrdEqual; case CmpInst::FCMP_OGE: return SPIRV::OpFOrdGreaterThanEqual; case CmpInst::FCMP_OGT: return SPIRV::OpFOrdGreaterThan; case CmpInst::FCMP_OLE: return SPIRV::OpFOrdLessThanEqual; case CmpInst::FCMP_OLT: return SPIRV::OpFOrdLessThan; case CmpInst::FCMP_ONE: return SPIRV::OpFOrdNotEqual; case CmpInst::FCMP_ORD: return SPIRV::OpOrdered; case CmpInst::FCMP_UEQ: return SPIRV::OpFUnordEqual; case CmpInst::FCMP_UGE: return SPIRV::OpFUnordGreaterThanEqual; case CmpInst::FCMP_UGT: return SPIRV::OpFUnordGreaterThan; case CmpInst::FCMP_ULE: return SPIRV::OpFUnordLessThanEqual; case CmpInst::FCMP_ULT: return SPIRV::OpFUnordLessThan; case CmpInst::FCMP_UNE: return SPIRV::OpFUnordNotEqual; case CmpInst::FCMP_UNO: return SPIRV::OpUnordered; default: llvm_unreachable("Unknown predicate type for FCmp"); } } static unsigned getICmpOpcode(unsigned PredNum) { auto Pred = static_cast(PredNum); switch (Pred) { case CmpInst::ICMP_EQ: return SPIRV::OpIEqual; case CmpInst::ICMP_NE: return SPIRV::OpINotEqual; case CmpInst::ICMP_SGE: return SPIRV::OpSGreaterThanEqual; case CmpInst::ICMP_SGT: return SPIRV::OpSGreaterThan; case CmpInst::ICMP_SLE: return SPIRV::OpSLessThanEqual; case CmpInst::ICMP_SLT: return SPIRV::OpSLessThan; case CmpInst::ICMP_UGE: return SPIRV::OpUGreaterThanEqual; case CmpInst::ICMP_UGT: return SPIRV::OpUGreaterThan; case CmpInst::ICMP_ULE: return SPIRV::OpULessThanEqual; case CmpInst::ICMP_ULT: return SPIRV::OpULessThan; default: llvm_unreachable("Unknown predicate type for ICmp"); } } static unsigned getPtrCmpOpcode(unsigned Pred) { switch (static_cast(Pred)) { case CmpInst::ICMP_EQ: return SPIRV::OpPtrEqual; case CmpInst::ICMP_NE: return SPIRV::OpPtrNotEqual; default: llvm_unreachable("Unknown predicate type for pointer comparison"); } } // Return the logical operation, or abort if none exists. static unsigned getBoolCmpOpcode(unsigned PredNum) { auto Pred = static_cast(PredNum); switch (Pred) { case CmpInst::ICMP_EQ: return SPIRV::OpLogicalEqual; case CmpInst::ICMP_NE: return SPIRV::OpLogicalNotEqual; default: llvm_unreachable("Unknown predicate type for Bool comparison"); } } bool SPIRVInstructionSelector::selectAnyOrAll(Register ResVReg, const SPIRVType *ResType, MachineInstr &I, unsigned OpAnyOrAll) const { assert(I.getNumOperands() == 3); assert(I.getOperand(2).isReg()); MachineBasicBlock &BB = *I.getParent(); Register InputRegister = I.getOperand(2).getReg(); SPIRVType *InputType = GR.getSPIRVTypeForVReg(InputRegister); if (!InputType) report_fatal_error("Input Type could not be determined."); bool IsBoolTy = GR.isScalarOrVectorOfType(InputRegister, SPIRV::OpTypeBool); bool IsVectorTy = InputType->getOpcode() == SPIRV::OpTypeVector; if (IsBoolTy && !IsVectorTy) { assert(ResVReg == I.getOperand(0).getReg()); return BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(TargetOpcode::COPY)) .addDef(ResVReg) .addUse(InputRegister) .constrainAllUses(TII, TRI, RBI); } bool IsFloatTy = GR.isScalarOrVectorOfType(InputRegister, SPIRV::OpTypeFloat); unsigned SpirvNotEqualId = IsFloatTy ? SPIRV::OpFOrdNotEqual : SPIRV::OpINotEqual; SPIRVType *SpvBoolScalarTy = GR.getOrCreateSPIRVBoolType(I, TII); SPIRVType *SpvBoolTy = SpvBoolScalarTy; Register NotEqualReg = ResVReg; if (IsVectorTy) { NotEqualReg = IsBoolTy ? InputRegister : MRI->createVirtualRegister(&SPIRV::IDRegClass); const unsigned NumElts = InputType->getOperand(2).getImm(); SpvBoolTy = GR.getOrCreateSPIRVVectorType(SpvBoolTy, NumElts, I, TII); } if (!IsBoolTy) { Register ConstZeroReg = IsFloatTy ? buildZerosValF(InputType, I) : buildZerosVal(InputType, I); BuildMI(BB, I, I.getDebugLoc(), TII.get(SpirvNotEqualId)) .addDef(NotEqualReg) .addUse(GR.getSPIRVTypeID(SpvBoolTy)) .addUse(InputRegister) .addUse(ConstZeroReg) .constrainAllUses(TII, TRI, RBI); } if (!IsVectorTy) return true; return BuildMI(BB, I, I.getDebugLoc(), TII.get(OpAnyOrAll)) .addDef(ResVReg) .addUse(GR.getSPIRVTypeID(SpvBoolScalarTy)) .addUse(NotEqualReg) .constrainAllUses(TII, TRI, RBI); } bool SPIRVInstructionSelector::selectAll(Register ResVReg, const SPIRVType *ResType, MachineInstr &I) const { return selectAnyOrAll(ResVReg, ResType, I, SPIRV::OpAll); } bool SPIRVInstructionSelector::selectAny(Register ResVReg, const SPIRVType *ResType, MachineInstr &I) const { return selectAnyOrAll(ResVReg, ResType, I, SPIRV::OpAny); } bool SPIRVInstructionSelector::selectFmix(Register ResVReg, const SPIRVType *ResType, MachineInstr &I) const { assert(I.getNumOperands() == 5); assert(I.getOperand(2).isReg()); assert(I.getOperand(3).isReg()); assert(I.getOperand(4).isReg()); MachineBasicBlock &BB = *I.getParent(); return BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpExtInst)) .addDef(ResVReg) .addUse(GR.getSPIRVTypeID(ResType)) .addImm(static_cast(SPIRV::InstructionSet::GLSL_std_450)) .addImm(GL::FMix) .addUse(I.getOperand(2).getReg()) .addUse(I.getOperand(3).getReg()) .addUse(I.getOperand(4).getReg()) .constrainAllUses(TII, TRI, RBI); } bool SPIRVInstructionSelector::selectRsqrt(Register ResVReg, const SPIRVType *ResType, MachineInstr &I) const { assert(I.getNumOperands() == 3); assert(I.getOperand(2).isReg()); MachineBasicBlock &BB = *I.getParent(); return BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpExtInst)) .addDef(ResVReg) .addUse(GR.getSPIRVTypeID(ResType)) .addImm(static_cast(SPIRV::InstructionSet::GLSL_std_450)) .addImm(GL::InverseSqrt) .addUse(I.getOperand(2).getReg()) .constrainAllUses(TII, TRI, RBI); } bool SPIRVInstructionSelector::selectBitreverse(Register ResVReg, const SPIRVType *ResType, MachineInstr &I) const { MachineBasicBlock &BB = *I.getParent(); return BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpBitReverse)) .addDef(ResVReg) .addUse(GR.getSPIRVTypeID(ResType)) .addUse(I.getOperand(1).getReg()) .constrainAllUses(TII, TRI, RBI); } bool SPIRVInstructionSelector::selectFreeze(Register ResVReg, const SPIRVType *ResType, MachineInstr &I) const { // There is no way to implement `freeze` correctly without support on SPIR-V // standard side, but we may at least address a simple (static) case when // undef/poison value presence is obvious. The main benefit of even // incomplete `freeze` support is preventing of translation from crashing due // to lack of support on legalization and instruction selection steps. if (!I.getOperand(0).isReg() || !I.getOperand(1).isReg()) return false; Register OpReg = I.getOperand(1).getReg(); if (MachineInstr *Def = MRI->getVRegDef(OpReg)) { Register Reg; switch (Def->getOpcode()) { case SPIRV::ASSIGN_TYPE: if (MachineInstr *AssignToDef = MRI->getVRegDef(Def->getOperand(1).getReg())) { if (AssignToDef->getOpcode() == TargetOpcode::G_IMPLICIT_DEF) Reg = Def->getOperand(2).getReg(); } break; case SPIRV::OpUndef: Reg = Def->getOperand(1).getReg(); break; } unsigned DestOpCode; if (Reg.isValid()) { DestOpCode = SPIRV::OpConstantNull; } else { DestOpCode = TargetOpcode::COPY; Reg = OpReg; } return BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(DestOpCode)) .addDef(I.getOperand(0).getReg()) .addUse(Reg) .constrainAllUses(TII, TRI, RBI); } return false; } bool SPIRVInstructionSelector::selectConstVector(Register ResVReg, const SPIRVType *ResType, MachineInstr &I) const { // TODO: only const case is supported for now. assert(std::all_of( I.operands_begin(), I.operands_end(), [this](const MachineOperand &MO) { if (MO.isDef()) return true; if (!MO.isReg()) return false; SPIRVType *ConstTy = this->MRI->getVRegDef(MO.getReg()); assert(ConstTy && ConstTy->getOpcode() == SPIRV::ASSIGN_TYPE && ConstTy->getOperand(1).isReg()); Register ConstReg = ConstTy->getOperand(1).getReg(); const MachineInstr *Const = this->MRI->getVRegDef(ConstReg); assert(Const); return (Const->getOpcode() == TargetOpcode::G_CONSTANT || Const->getOpcode() == TargetOpcode::G_FCONSTANT); })); auto MIB = BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(SPIRV::OpConstantComposite)) .addDef(ResVReg) .addUse(GR.getSPIRVTypeID(ResType)); for (unsigned i = I.getNumExplicitDefs(); i < I.getNumExplicitOperands(); ++i) MIB.addUse(I.getOperand(i).getReg()); return MIB.constrainAllUses(TII, TRI, RBI); } static unsigned getArrayComponentCount(MachineRegisterInfo *MRI, const SPIRVType *ResType) { Register OpReg = ResType->getOperand(2).getReg(); SPIRVType *OpDef = MRI->getVRegDef(OpReg); if (!OpDef) return 0; if (OpDef->getOpcode() == SPIRV::ASSIGN_TYPE && OpDef->getOperand(1).isReg()) { if (SPIRVType *RefDef = MRI->getVRegDef(OpDef->getOperand(1).getReg())) OpDef = RefDef; } unsigned N = OpDef->getOpcode() == TargetOpcode::G_CONSTANT ? OpDef->getOperand(1).getCImm()->getValue().getZExtValue() : 0; return N; } // Return true if the type represents a constant register static bool isConstReg(MachineRegisterInfo *MRI, SPIRVType *OpDef, SmallPtrSet &Visited) { if (OpDef->getOpcode() == SPIRV::ASSIGN_TYPE && OpDef->getOperand(1).isReg()) { if (SPIRVType *RefDef = MRI->getVRegDef(OpDef->getOperand(1).getReg())) OpDef = RefDef; } if (Visited.contains(OpDef)) return true; Visited.insert(OpDef); unsigned Opcode = OpDef->getOpcode(); switch (Opcode) { case TargetOpcode::G_CONSTANT: case TargetOpcode::G_FCONSTANT: return true; case TargetOpcode::G_INTRINSIC: case TargetOpcode::G_INTRINSIC_W_SIDE_EFFECTS: case TargetOpcode::G_INTRINSIC_CONVERGENT_W_SIDE_EFFECTS: return cast(*OpDef).getIntrinsicID() == Intrinsic::spv_const_composite; case TargetOpcode::G_BUILD_VECTOR: case TargetOpcode::G_SPLAT_VECTOR: { for (unsigned i = OpDef->getNumExplicitDefs(); i < OpDef->getNumOperands(); i++) { SPIRVType *OpNestedDef = OpDef->getOperand(i).isReg() ? MRI->getVRegDef(OpDef->getOperand(i).getReg()) : nullptr; if (OpNestedDef && !isConstReg(MRI, OpNestedDef, Visited)) return false; } return true; } } return false; } // Return true if the virtual register represents a constant static bool isConstReg(MachineRegisterInfo *MRI, Register OpReg) { SmallPtrSet Visited; if (SPIRVType *OpDef = MRI->getVRegDef(OpReg)) return isConstReg(MRI, OpDef, Visited); return false; } bool SPIRVInstructionSelector::selectSplatVector(Register ResVReg, const SPIRVType *ResType, MachineInstr &I) const { unsigned N = 0; if (ResType->getOpcode() == SPIRV::OpTypeVector) N = GR.getScalarOrVectorComponentCount(ResType); else if (ResType->getOpcode() == SPIRV::OpTypeArray) N = getArrayComponentCount(MRI, ResType); else report_fatal_error("Cannot select G_SPLAT_VECTOR with a non-vector result"); unsigned OpIdx = I.getNumExplicitDefs(); if (!I.getOperand(OpIdx).isReg()) report_fatal_error("Unexpected argument in G_SPLAT_VECTOR"); // check if we may construct a constant vector Register OpReg = I.getOperand(OpIdx).getReg(); bool IsConst = isConstReg(MRI, OpReg); if (!IsConst && N < 2) report_fatal_error( "There must be at least two constituent operands in a vector"); auto MIB = BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(IsConst ? SPIRV::OpConstantComposite : SPIRV::OpCompositeConstruct)) .addDef(ResVReg) .addUse(GR.getSPIRVTypeID(ResType)); for (unsigned i = 0; i < N; ++i) MIB.addUse(OpReg); return MIB.constrainAllUses(TII, TRI, RBI); } bool SPIRVInstructionSelector::selectCmp(Register ResVReg, const SPIRVType *ResType, unsigned CmpOpc, MachineInstr &I) const { Register Cmp0 = I.getOperand(2).getReg(); Register Cmp1 = I.getOperand(3).getReg(); assert(GR.getSPIRVTypeForVReg(Cmp0)->getOpcode() == GR.getSPIRVTypeForVReg(Cmp1)->getOpcode() && "CMP operands should have the same type"); return BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(CmpOpc)) .addDef(ResVReg) .addUse(GR.getSPIRVTypeID(ResType)) .addUse(Cmp0) .addUse(Cmp1) .constrainAllUses(TII, TRI, RBI); } bool SPIRVInstructionSelector::selectICmp(Register ResVReg, const SPIRVType *ResType, MachineInstr &I) const { auto Pred = I.getOperand(1).getPredicate(); unsigned CmpOpc; Register CmpOperand = I.getOperand(2).getReg(); if (GR.isScalarOfType(CmpOperand, SPIRV::OpTypePointer)) CmpOpc = getPtrCmpOpcode(Pred); else if (GR.isScalarOrVectorOfType(CmpOperand, SPIRV::OpTypeBool)) CmpOpc = getBoolCmpOpcode(Pred); else CmpOpc = getICmpOpcode(Pred); return selectCmp(ResVReg, ResType, CmpOpc, I); } void SPIRVInstructionSelector::renderFImm32(MachineInstrBuilder &MIB, const MachineInstr &I, int OpIdx) const { assert(I.getOpcode() == TargetOpcode::G_FCONSTANT && OpIdx == -1 && "Expected G_FCONSTANT"); const ConstantFP *FPImm = I.getOperand(1).getFPImm(); addNumImm(FPImm->getValueAPF().bitcastToAPInt(), MIB); } void SPIRVInstructionSelector::renderImm32(MachineInstrBuilder &MIB, const MachineInstr &I, int OpIdx) const { assert(I.getOpcode() == TargetOpcode::G_CONSTANT && OpIdx == -1 && "Expected G_CONSTANT"); addNumImm(I.getOperand(1).getCImm()->getValue(), MIB); } Register SPIRVInstructionSelector::buildI32Constant(uint32_t Val, MachineInstr &I, const SPIRVType *ResType) const { Type *LLVMTy = IntegerType::get(GR.CurMF->getFunction().getContext(), 32); const SPIRVType *SpvI32Ty = ResType ? ResType : GR.getOrCreateSPIRVIntegerType(32, I, TII); // Find a constant in DT or build a new one. auto ConstInt = ConstantInt::get(LLVMTy, Val); Register NewReg = GR.find(ConstInt, GR.CurMF); if (!NewReg.isValid()) { NewReg = MRI->createGenericVirtualRegister(LLT::scalar(32)); GR.add(ConstInt, GR.CurMF, NewReg); MachineInstr *MI; MachineBasicBlock &BB = *I.getParent(); if (Val == 0) { MI = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpConstantNull)) .addDef(NewReg) .addUse(GR.getSPIRVTypeID(SpvI32Ty)); } else { MI = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpConstantI)) .addDef(NewReg) .addUse(GR.getSPIRVTypeID(SpvI32Ty)) .addImm(APInt(32, Val).getZExtValue()); } constrainSelectedInstRegOperands(*MI, TII, TRI, RBI); } return NewReg; } bool SPIRVInstructionSelector::selectFCmp(Register ResVReg, const SPIRVType *ResType, MachineInstr &I) const { unsigned CmpOp = getFCmpOpcode(I.getOperand(1).getPredicate()); return selectCmp(ResVReg, ResType, CmpOp, I); } Register SPIRVInstructionSelector::buildZerosVal(const SPIRVType *ResType, MachineInstr &I) const { // OpenCL uses nulls for Zero. In HLSL we don't use null constants. bool ZeroAsNull = STI.isOpenCLEnv(); if (ResType->getOpcode() == SPIRV::OpTypeVector) return GR.getOrCreateConstVector(0UL, I, ResType, TII, ZeroAsNull); return GR.getOrCreateConstInt(0, I, ResType, TII, ZeroAsNull); } static APFloat getZeroFP(const Type *LLVMFloatTy) { if (!LLVMFloatTy) return APFloat::getZero(APFloat::IEEEsingle()); switch (LLVMFloatTy->getScalarType()->getTypeID()) { case Type::HalfTyID: return APFloat::getZero(APFloat::IEEEhalf()); default: case Type::FloatTyID: return APFloat::getZero(APFloat::IEEEsingle()); case Type::DoubleTyID: return APFloat::getZero(APFloat::IEEEdouble()); } } Register SPIRVInstructionSelector::buildZerosValF(const SPIRVType *ResType, MachineInstr &I) const { // OpenCL uses nulls for Zero. In HLSL we don't use null constants. bool ZeroAsNull = STI.isOpenCLEnv(); APFloat VZero = getZeroFP(GR.getTypeForSPIRVType(ResType)); if (ResType->getOpcode() == SPIRV::OpTypeVector) return GR.getOrCreateConstVector(VZero, I, ResType, TII, ZeroAsNull); return GR.getOrCreateConstFP(VZero, I, ResType, TII, ZeroAsNull); } Register SPIRVInstructionSelector::buildOnesVal(bool AllOnes, const SPIRVType *ResType, MachineInstr &I) const { unsigned BitWidth = GR.getScalarOrVectorBitWidth(ResType); APInt One = AllOnes ? APInt::getAllOnes(BitWidth) : APInt::getOneBitSet(BitWidth, 0); if (ResType->getOpcode() == SPIRV::OpTypeVector) return GR.getOrCreateConstVector(One.getZExtValue(), I, ResType, TII); return GR.getOrCreateConstInt(One.getZExtValue(), I, ResType, TII); } bool SPIRVInstructionSelector::selectSelect(Register ResVReg, const SPIRVType *ResType, MachineInstr &I, bool IsSigned) const { // To extend a bool, we need to use OpSelect between constants. Register ZeroReg = buildZerosVal(ResType, I); Register OneReg = buildOnesVal(IsSigned, ResType, I); bool IsScalarBool = GR.isScalarOfType(I.getOperand(1).getReg(), SPIRV::OpTypeBool); unsigned Opcode = IsScalarBool ? SPIRV::OpSelectSISCond : SPIRV::OpSelectSIVCond; return BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(Opcode)) .addDef(ResVReg) .addUse(GR.getSPIRVTypeID(ResType)) .addUse(I.getOperand(1).getReg()) .addUse(OneReg) .addUse(ZeroReg) .constrainAllUses(TII, TRI, RBI); } bool SPIRVInstructionSelector::selectIToF(Register ResVReg, const SPIRVType *ResType, MachineInstr &I, bool IsSigned, unsigned Opcode) const { Register SrcReg = I.getOperand(1).getReg(); // We can convert bool value directly to float type without OpConvert*ToF, // however the translator generates OpSelect+OpConvert*ToF, so we do the same. if (GR.isScalarOrVectorOfType(I.getOperand(1).getReg(), SPIRV::OpTypeBool)) { unsigned BitWidth = GR.getScalarOrVectorBitWidth(ResType); SPIRVType *TmpType = GR.getOrCreateSPIRVIntegerType(BitWidth, I, TII); if (ResType->getOpcode() == SPIRV::OpTypeVector) { const unsigned NumElts = ResType->getOperand(2).getImm(); TmpType = GR.getOrCreateSPIRVVectorType(TmpType, NumElts, I, TII); } SrcReg = MRI->createVirtualRegister(&SPIRV::IDRegClass); selectSelect(SrcReg, TmpType, I, false); } return selectUnOpWithSrc(ResVReg, ResType, I, SrcReg, Opcode); } bool SPIRVInstructionSelector::selectExt(Register ResVReg, const SPIRVType *ResType, MachineInstr &I, bool IsSigned) const { Register SrcReg = I.getOperand(1).getReg(); if (GR.isScalarOrVectorOfType(SrcReg, SPIRV::OpTypeBool)) return selectSelect(ResVReg, ResType, I, IsSigned); SPIRVType *SrcType = GR.getSPIRVTypeForVReg(SrcReg); if (SrcType == ResType) return BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(TargetOpcode::COPY)) .addDef(ResVReg) .addUse(SrcReg) .constrainAllUses(TII, TRI, RBI); unsigned Opcode = IsSigned ? SPIRV::OpSConvert : SPIRV::OpUConvert; return selectUnOp(ResVReg, ResType, I, Opcode); } bool SPIRVInstructionSelector::selectIntToBool(Register IntReg, Register ResVReg, MachineInstr &I, const SPIRVType *IntTy, const SPIRVType *BoolTy) const { // To truncate to a bool, we use OpBitwiseAnd 1 and OpINotEqual to zero. Register BitIntReg = MRI->createVirtualRegister(&SPIRV::IDRegClass); bool IsVectorTy = IntTy->getOpcode() == SPIRV::OpTypeVector; unsigned Opcode = IsVectorTy ? SPIRV::OpBitwiseAndV : SPIRV::OpBitwiseAndS; Register Zero = buildZerosVal(IntTy, I); Register One = buildOnesVal(false, IntTy, I); MachineBasicBlock &BB = *I.getParent(); BuildMI(BB, I, I.getDebugLoc(), TII.get(Opcode)) .addDef(BitIntReg) .addUse(GR.getSPIRVTypeID(IntTy)) .addUse(IntReg) .addUse(One) .constrainAllUses(TII, TRI, RBI); return BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpINotEqual)) .addDef(ResVReg) .addUse(GR.getSPIRVTypeID(BoolTy)) .addUse(BitIntReg) .addUse(Zero) .constrainAllUses(TII, TRI, RBI); } bool SPIRVInstructionSelector::selectTrunc(Register ResVReg, const SPIRVType *ResType, MachineInstr &I) const { Register IntReg = I.getOperand(1).getReg(); const SPIRVType *ArgType = GR.getSPIRVTypeForVReg(IntReg); if (GR.isScalarOrVectorOfType(ResVReg, SPIRV::OpTypeBool)) return selectIntToBool(IntReg, ResVReg, I, ArgType, ResType); if (ArgType == ResType) return BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(TargetOpcode::COPY)) .addDef(ResVReg) .addUse(IntReg) .constrainAllUses(TII, TRI, RBI); bool IsSigned = GR.isScalarOrVectorSigned(ResType); unsigned Opcode = IsSigned ? SPIRV::OpSConvert : SPIRV::OpUConvert; return selectUnOp(ResVReg, ResType, I, Opcode); } bool SPIRVInstructionSelector::selectConst(Register ResVReg, const SPIRVType *ResType, const APInt &Imm, MachineInstr &I) const { unsigned TyOpcode = ResType->getOpcode(); assert(TyOpcode != SPIRV::OpTypePointer || Imm.isZero()); MachineBasicBlock &BB = *I.getParent(); if ((TyOpcode == SPIRV::OpTypePointer || TyOpcode == SPIRV::OpTypeEvent) && Imm.isZero()) return BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpConstantNull)) .addDef(ResVReg) .addUse(GR.getSPIRVTypeID(ResType)) .constrainAllUses(TII, TRI, RBI); if (TyOpcode == SPIRV::OpTypeInt) { assert(Imm.getBitWidth() <= 64 && "Unsupported integer width!"); Register Reg = GR.getOrCreateConstInt(Imm.getZExtValue(), I, ResType, TII); if (Reg == ResVReg) return true; return BuildMI(BB, I, I.getDebugLoc(), TII.get(TargetOpcode::COPY)) .addDef(ResVReg) .addUse(Reg) .constrainAllUses(TII, TRI, RBI); } auto MIB = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpConstantI)) .addDef(ResVReg) .addUse(GR.getSPIRVTypeID(ResType)); // <=32-bit integers should be caught by the sdag pattern. assert(Imm.getBitWidth() > 32); addNumImm(Imm, MIB); return MIB.constrainAllUses(TII, TRI, RBI); } bool SPIRVInstructionSelector::selectOpUndef(Register ResVReg, const SPIRVType *ResType, MachineInstr &I) const { return BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(SPIRV::OpUndef)) .addDef(ResVReg) .addUse(GR.getSPIRVTypeID(ResType)) .constrainAllUses(TII, TRI, RBI); } static bool isImm(const MachineOperand &MO, MachineRegisterInfo *MRI) { assert(MO.isReg()); const SPIRVType *TypeInst = MRI->getVRegDef(MO.getReg()); if (TypeInst->getOpcode() == SPIRV::ASSIGN_TYPE) { assert(TypeInst->getOperand(1).isReg()); MachineInstr *ImmInst = MRI->getVRegDef(TypeInst->getOperand(1).getReg()); return ImmInst->getOpcode() == TargetOpcode::G_CONSTANT; } return TypeInst->getOpcode() == SPIRV::OpConstantI; } static int64_t foldImm(const MachineOperand &MO, MachineRegisterInfo *MRI) { const SPIRVType *TypeInst = MRI->getVRegDef(MO.getReg()); if (TypeInst->getOpcode() == SPIRV::OpConstantI) return TypeInst->getOperand(2).getImm(); MachineInstr *ImmInst = MRI->getVRegDef(TypeInst->getOperand(1).getReg()); assert(ImmInst->getOpcode() == TargetOpcode::G_CONSTANT); return ImmInst->getOperand(1).getCImm()->getZExtValue(); } bool SPIRVInstructionSelector::selectInsertVal(Register ResVReg, const SPIRVType *ResType, MachineInstr &I) const { MachineBasicBlock &BB = *I.getParent(); auto MIB = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpCompositeInsert)) .addDef(ResVReg) .addUse(GR.getSPIRVTypeID(ResType)) // object to insert .addUse(I.getOperand(3).getReg()) // composite to insert into .addUse(I.getOperand(2).getReg()); for (unsigned i = 4; i < I.getNumOperands(); i++) MIB.addImm(foldImm(I.getOperand(i), MRI)); return MIB.constrainAllUses(TII, TRI, RBI); } bool SPIRVInstructionSelector::selectExtractVal(Register ResVReg, const SPIRVType *ResType, MachineInstr &I) const { MachineBasicBlock &BB = *I.getParent(); auto MIB = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpCompositeExtract)) .addDef(ResVReg) .addUse(GR.getSPIRVTypeID(ResType)) .addUse(I.getOperand(2).getReg()); for (unsigned i = 3; i < I.getNumOperands(); i++) MIB.addImm(foldImm(I.getOperand(i), MRI)); return MIB.constrainAllUses(TII, TRI, RBI); } bool SPIRVInstructionSelector::selectInsertElt(Register ResVReg, const SPIRVType *ResType, MachineInstr &I) const { if (isImm(I.getOperand(4), MRI)) return selectInsertVal(ResVReg, ResType, I); MachineBasicBlock &BB = *I.getParent(); return BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpVectorInsertDynamic)) .addDef(ResVReg) .addUse(GR.getSPIRVTypeID(ResType)) .addUse(I.getOperand(2).getReg()) .addUse(I.getOperand(3).getReg()) .addUse(I.getOperand(4).getReg()) .constrainAllUses(TII, TRI, RBI); } bool SPIRVInstructionSelector::selectExtractElt(Register ResVReg, const SPIRVType *ResType, MachineInstr &I) const { if (isImm(I.getOperand(3), MRI)) return selectExtractVal(ResVReg, ResType, I); MachineBasicBlock &BB = *I.getParent(); return BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpVectorExtractDynamic)) .addDef(ResVReg) .addUse(GR.getSPIRVTypeID(ResType)) .addUse(I.getOperand(2).getReg()) .addUse(I.getOperand(3).getReg()) .constrainAllUses(TII, TRI, RBI); } bool SPIRVInstructionSelector::selectGEP(Register ResVReg, const SPIRVType *ResType, MachineInstr &I) const { const bool IsGEPInBounds = I.getOperand(2).getImm(); // OpAccessChain could be used for OpenCL, but the SPIRV-LLVM Translator only // relies on PtrAccessChain, so we'll try not to deviate. For Vulkan however, // we have to use Op[InBounds]AccessChain. const unsigned Opcode = STI.isVulkanEnv() ? (IsGEPInBounds ? SPIRV::OpInBoundsAccessChain : SPIRV::OpAccessChain) : (IsGEPInBounds ? SPIRV::OpInBoundsPtrAccessChain : SPIRV::OpPtrAccessChain); auto Res = BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(Opcode)) .addDef(ResVReg) .addUse(GR.getSPIRVTypeID(ResType)) // Object to get a pointer to. .addUse(I.getOperand(3).getReg()); // Adding indices. const unsigned StartingIndex = (Opcode == SPIRV::OpAccessChain || Opcode == SPIRV::OpInBoundsAccessChain) ? 5 : 4; for (unsigned i = StartingIndex; i < I.getNumExplicitOperands(); ++i) Res.addUse(I.getOperand(i).getReg()); return Res.constrainAllUses(TII, TRI, RBI); } // Maybe wrap a value into OpSpecConstantOp bool SPIRVInstructionSelector::wrapIntoSpecConstantOp( MachineInstr &I, SmallVector &CompositeArgs) const { bool Result = true; unsigned Lim = I.getNumExplicitOperands(); for (unsigned i = I.getNumExplicitDefs() + 1; i < Lim; ++i) { Register OpReg = I.getOperand(i).getReg(); SPIRVType *OpDefine = MRI->getVRegDef(OpReg); SPIRVType *OpType = GR.getSPIRVTypeForVReg(OpReg); SmallPtrSet Visited; if (!OpDefine || !OpType || isConstReg(MRI, OpDefine, Visited) || OpDefine->getOpcode() == TargetOpcode::G_ADDRSPACE_CAST || GR.isAggregateType(OpType)) { // The case of G_ADDRSPACE_CAST inside spv_const_composite() is processed // by selectAddrSpaceCast() CompositeArgs.push_back(OpReg); continue; } MachineFunction *MF = I.getMF(); Register WrapReg = GR.find(OpDefine, MF); if (WrapReg.isValid()) { CompositeArgs.push_back(WrapReg); continue; } // Create a new register for the wrapper WrapReg = MRI->createVirtualRegister(&SPIRV::IDRegClass); GR.add(OpDefine, MF, WrapReg); CompositeArgs.push_back(WrapReg); // Decorate the wrapper register and generate a new instruction MRI->setType(WrapReg, LLT::pointer(0, 32)); GR.assignSPIRVTypeToVReg(OpType, WrapReg, *MF); MachineBasicBlock &BB = *I.getParent(); Result = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpSpecConstantOp)) .addDef(WrapReg) .addUse(GR.getSPIRVTypeID(OpType)) .addImm(static_cast(SPIRV::Opcode::Bitcast)) .addUse(OpReg) .constrainAllUses(TII, TRI, RBI); if (!Result) break; } return Result; } bool SPIRVInstructionSelector::selectIntrinsic(Register ResVReg, const SPIRVType *ResType, MachineInstr &I) const { MachineBasicBlock &BB = *I.getParent(); Intrinsic::ID IID = cast(I).getIntrinsicID(); switch (IID) { case Intrinsic::spv_load: return selectLoad(ResVReg, ResType, I); case Intrinsic::spv_store: return selectStore(I); case Intrinsic::spv_extractv: return selectExtractVal(ResVReg, ResType, I); case Intrinsic::spv_insertv: return selectInsertVal(ResVReg, ResType, I); case Intrinsic::spv_extractelt: return selectExtractElt(ResVReg, ResType, I); case Intrinsic::spv_insertelt: return selectInsertElt(ResVReg, ResType, I); case Intrinsic::spv_gep: return selectGEP(ResVReg, ResType, I); case Intrinsic::spv_unref_global: case Intrinsic::spv_init_global: { MachineInstr *MI = MRI->getVRegDef(I.getOperand(1).getReg()); MachineInstr *Init = I.getNumExplicitOperands() > 2 ? MRI->getVRegDef(I.getOperand(2).getReg()) : nullptr; assert(MI); return selectGlobalValue(MI->getOperand(0).getReg(), *MI, Init); } case Intrinsic::spv_undef: { auto MIB = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpUndef)) .addDef(ResVReg) .addUse(GR.getSPIRVTypeID(ResType)); return MIB.constrainAllUses(TII, TRI, RBI); } case Intrinsic::spv_const_composite: { // If no values are attached, the composite is null constant. bool IsNull = I.getNumExplicitDefs() + 1 == I.getNumExplicitOperands(); // Select a proper instruction. unsigned Opcode = SPIRV::OpConstantNull; SmallVector CompositeArgs; if (!IsNull) { Opcode = SPIRV::OpConstantComposite; if (!wrapIntoSpecConstantOp(I, CompositeArgs)) return false; } auto MIB = BuildMI(BB, I, I.getDebugLoc(), TII.get(Opcode)) .addDef(ResVReg) .addUse(GR.getSPIRVTypeID(ResType)); // skip type MD node we already used when generated assign.type for this if (!IsNull) { for (Register OpReg : CompositeArgs) MIB.addUse(OpReg); } return MIB.constrainAllUses(TII, TRI, RBI); } case Intrinsic::spv_assign_name: { auto MIB = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpName)); MIB.addUse(I.getOperand(I.getNumExplicitDefs() + 1).getReg()); for (unsigned i = I.getNumExplicitDefs() + 2; i < I.getNumExplicitOperands(); ++i) { MIB.addImm(I.getOperand(i).getImm()); } return MIB.constrainAllUses(TII, TRI, RBI); } case Intrinsic::spv_switch: { auto MIB = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpSwitch)); for (unsigned i = 1; i < I.getNumExplicitOperands(); ++i) { if (I.getOperand(i).isReg()) MIB.addReg(I.getOperand(i).getReg()); else if (I.getOperand(i).isCImm()) addNumImm(I.getOperand(i).getCImm()->getValue(), MIB); else if (I.getOperand(i).isMBB()) MIB.addMBB(I.getOperand(i).getMBB()); else llvm_unreachable("Unexpected OpSwitch operand"); } return MIB.constrainAllUses(TII, TRI, RBI); } case Intrinsic::spv_cmpxchg: return selectAtomicCmpXchg(ResVReg, ResType, I); case Intrinsic::spv_unreachable: BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpUnreachable)); break; case Intrinsic::spv_alloca: return selectFrameIndex(ResVReg, ResType, I); case Intrinsic::spv_alloca_array: return selectAllocaArray(ResVReg, ResType, I); case Intrinsic::spv_assume: if (STI.canUseExtension(SPIRV::Extension::SPV_KHR_expect_assume)) BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpAssumeTrueKHR)) .addUse(I.getOperand(1).getReg()); break; case Intrinsic::spv_expect: if (STI.canUseExtension(SPIRV::Extension::SPV_KHR_expect_assume)) BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpExpectKHR)) .addDef(ResVReg) .addUse(GR.getSPIRVTypeID(ResType)) .addUse(I.getOperand(2).getReg()) .addUse(I.getOperand(3).getReg()); break; case Intrinsic::spv_thread_id: return selectSpvThreadId(ResVReg, ResType, I); case Intrinsic::spv_all: return selectAll(ResVReg, ResType, I); case Intrinsic::spv_any: return selectAny(ResVReg, ResType, I); case Intrinsic::spv_lerp: return selectFmix(ResVReg, ResType, I); case Intrinsic::spv_rsqrt: return selectRsqrt(ResVReg, ResType, I); case Intrinsic::spv_lifetime_start: case Intrinsic::spv_lifetime_end: { unsigned Op = IID == Intrinsic::spv_lifetime_start ? SPIRV::OpLifetimeStart : SPIRV::OpLifetimeStop; int64_t Size = I.getOperand(I.getNumExplicitDefs() + 1).getImm(); Register PtrReg = I.getOperand(I.getNumExplicitDefs() + 2).getReg(); unsigned PonteeOpType = GR.getPointeeTypeOp(PtrReg); bool IsNonvoidPtr = PonteeOpType != 0 && PonteeOpType != SPIRV::OpTypeVoid; if (Size == -1 || IsNonvoidPtr) Size = 0; BuildMI(BB, I, I.getDebugLoc(), TII.get(Op)).addUse(PtrReg).addImm(Size); } break; default: { std::string DiagMsg; raw_string_ostream OS(DiagMsg); I.print(OS); DiagMsg = "Intrinsic selection not implemented: " + DiagMsg; report_fatal_error(DiagMsg.c_str(), false); } } return true; } bool SPIRVInstructionSelector::selectAllocaArray(Register ResVReg, const SPIRVType *ResType, MachineInstr &I) const { // there was an allocation size parameter to the allocation instruction // that is not 1 MachineBasicBlock &BB = *I.getParent(); return BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpVariableLengthArrayINTEL)) .addDef(ResVReg) .addUse(GR.getSPIRVTypeID(ResType)) .addUse(I.getOperand(2).getReg()) .constrainAllUses(TII, TRI, RBI); } bool SPIRVInstructionSelector::selectFrameIndex(Register ResVReg, const SPIRVType *ResType, MachineInstr &I) const { // Change order of instructions if needed: all OpVariable instructions in a // function must be the first instructions in the first block MachineFunction *MF = I.getParent()->getParent(); MachineBasicBlock *MBB = &MF->front(); auto It = MBB->SkipPHIsAndLabels(MBB->begin()), E = MBB->end(); bool IsHeader = false; unsigned Opcode; for (; It != E && It != I; ++It) { Opcode = It->getOpcode(); if (Opcode == SPIRV::OpFunction || Opcode == SPIRV::OpFunctionParameter) { IsHeader = true; } else if (IsHeader && !(Opcode == SPIRV::ASSIGN_TYPE || Opcode == SPIRV::OpLabel)) { ++It; break; } } return BuildMI(*MBB, It, It->getDebugLoc(), TII.get(SPIRV::OpVariable)) .addDef(ResVReg) .addUse(GR.getSPIRVTypeID(ResType)) .addImm(static_cast(SPIRV::StorageClass::Function)) .constrainAllUses(TII, TRI, RBI); } bool SPIRVInstructionSelector::selectBranch(MachineInstr &I) const { // InstructionSelector walks backwards through the instructions. We can use // both a G_BR and a G_BRCOND to create an OpBranchConditional. We hit G_BR // first, so can generate an OpBranchConditional here. If there is no // G_BRCOND, we just use OpBranch for a regular unconditional branch. const MachineInstr *PrevI = I.getPrevNode(); MachineBasicBlock &MBB = *I.getParent(); if (PrevI != nullptr && PrevI->getOpcode() == TargetOpcode::G_BRCOND) { return BuildMI(MBB, I, I.getDebugLoc(), TII.get(SPIRV::OpBranchConditional)) .addUse(PrevI->getOperand(0).getReg()) .addMBB(PrevI->getOperand(1).getMBB()) .addMBB(I.getOperand(0).getMBB()) .constrainAllUses(TII, TRI, RBI); } return BuildMI(MBB, I, I.getDebugLoc(), TII.get(SPIRV::OpBranch)) .addMBB(I.getOperand(0).getMBB()) .constrainAllUses(TII, TRI, RBI); } bool SPIRVInstructionSelector::selectBranchCond(MachineInstr &I) const { // InstructionSelector walks backwards through the instructions. For an // explicit conditional branch with no fallthrough, we use both a G_BR and a // G_BRCOND to create an OpBranchConditional. We should hit G_BR first, and // generate the OpBranchConditional in selectBranch above. // // If an OpBranchConditional has been generated, we simply return, as the work // is alread done. If there is no OpBranchConditional, LLVM must be relying on // implicit fallthrough to the next basic block, so we need to create an // OpBranchConditional with an explicit "false" argument pointing to the next // basic block that LLVM would fall through to. const MachineInstr *NextI = I.getNextNode(); // Check if this has already been successfully selected. if (NextI != nullptr && NextI->getOpcode() == SPIRV::OpBranchConditional) return true; // Must be relying on implicit block fallthrough, so generate an // OpBranchConditional with the "next" basic block as the "false" target. MachineBasicBlock &MBB = *I.getParent(); unsigned NextMBBNum = MBB.getNextNode()->getNumber(); MachineBasicBlock *NextMBB = I.getMF()->getBlockNumbered(NextMBBNum); return BuildMI(MBB, I, I.getDebugLoc(), TII.get(SPIRV::OpBranchConditional)) .addUse(I.getOperand(0).getReg()) .addMBB(I.getOperand(1).getMBB()) .addMBB(NextMBB) .constrainAllUses(TII, TRI, RBI); } bool SPIRVInstructionSelector::selectPhi(Register ResVReg, const SPIRVType *ResType, MachineInstr &I) const { auto MIB = BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(SPIRV::OpPhi)) .addDef(ResVReg) .addUse(GR.getSPIRVTypeID(ResType)); const unsigned NumOps = I.getNumOperands(); for (unsigned i = 1; i < NumOps; i += 2) { MIB.addUse(I.getOperand(i + 0).getReg()); MIB.addMBB(I.getOperand(i + 1).getMBB()); } return MIB.constrainAllUses(TII, TRI, RBI); } bool SPIRVInstructionSelector::selectGlobalValue( Register ResVReg, MachineInstr &I, const MachineInstr *Init) const { // FIXME: don't use MachineIRBuilder here, replace it with BuildMI. MachineIRBuilder MIRBuilder(I); const GlobalValue *GV = I.getOperand(1).getGlobal(); Type *GVType = toTypedPointer(GR.getDeducedGlobalValueType(GV)); SPIRVType *PointerBaseType; if (GVType->isArrayTy()) { SPIRVType *ArrayElementType = GR.getOrCreateSPIRVType(GVType->getArrayElementType(), MIRBuilder, SPIRV::AccessQualifier::ReadWrite, false); PointerBaseType = GR.getOrCreateSPIRVArrayType( ArrayElementType, GVType->getArrayNumElements(), I, TII); } else { PointerBaseType = GR.getOrCreateSPIRVType( GVType, MIRBuilder, SPIRV::AccessQualifier::ReadWrite, false); } SPIRVType *ResType = GR.getOrCreateSPIRVPointerType( PointerBaseType, I, TII, addressSpaceToStorageClass(GV->getAddressSpace(), STI)); std::string GlobalIdent; if (!GV->hasName()) { unsigned &ID = UnnamedGlobalIDs[GV]; if (ID == 0) ID = UnnamedGlobalIDs.size(); GlobalIdent = "__unnamed_" + Twine(ID).str(); } else { GlobalIdent = GV->getGlobalIdentifier(); } // Behaviour of functions as operands depends on availability of the // corresponding extension (SPV_INTEL_function_pointers): // - If there is an extension to operate with functions as operands: // We create a proper constant operand and evaluate a correct type for a // function pointer. // - Without the required extension: // We have functions as operands in tests with blocks of instruction e.g. in // transcoding/global_block.ll. These operands are not used and should be // substituted by zero constants. Their type is expected to be always // OpTypePointer Function %uchar. if (isa(GV)) { const Constant *ConstVal = GV; MachineBasicBlock &BB = *I.getParent(); Register NewReg = GR.find(ConstVal, GR.CurMF); if (!NewReg.isValid()) { Register NewReg = ResVReg; GR.add(ConstVal, GR.CurMF, NewReg); const Function *GVFun = STI.canUseExtension(SPIRV::Extension::SPV_INTEL_function_pointers) ? dyn_cast(GV) : nullptr; if (GVFun) { // References to a function via function pointers generate virtual // registers without a definition. We will resolve it later, during // module analysis stage. MachineRegisterInfo *MRI = MIRBuilder.getMRI(); Register FuncVReg = MRI->createGenericVirtualRegister(LLT::scalar(32)); MRI->setRegClass(FuncVReg, &SPIRV::IDRegClass); MachineInstrBuilder MB = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpConstantFunctionPointerINTEL)) .addDef(NewReg) .addUse(GR.getSPIRVTypeID(ResType)) .addUse(FuncVReg); // mapping the function pointer to the used Function GR.recordFunctionPointer(&MB.getInstr()->getOperand(2), GVFun); return MB.constrainAllUses(TII, TRI, RBI); } return BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpConstantNull)) .addDef(NewReg) .addUse(GR.getSPIRVTypeID(ResType)) .constrainAllUses(TII, TRI, RBI); } assert(NewReg != ResVReg); return BuildMI(BB, I, I.getDebugLoc(), TII.get(TargetOpcode::COPY)) .addDef(ResVReg) .addUse(NewReg) .constrainAllUses(TII, TRI, RBI); } auto GlobalVar = cast(GV); assert(GlobalVar->getName() != "llvm.global.annotations"); bool HasInit = GlobalVar->hasInitializer() && !isa(GlobalVar->getInitializer()); // Skip empty declaration for GVs with initilaizers till we get the decl with // passed initializer. if (HasInit && !Init) return true; unsigned AddrSpace = GV->getAddressSpace(); SPIRV::StorageClass::StorageClass Storage = addressSpaceToStorageClass(AddrSpace, STI); bool HasLnkTy = GV->getLinkage() != GlobalValue::InternalLinkage && Storage != SPIRV::StorageClass::Function; SPIRV::LinkageType::LinkageType LnkType = (GV->isDeclaration() || GV->hasAvailableExternallyLinkage()) ? SPIRV::LinkageType::Import : (GV->getLinkage() == GlobalValue::LinkOnceODRLinkage && STI.canUseExtension(SPIRV::Extension::SPV_KHR_linkonce_odr) ? SPIRV::LinkageType::LinkOnceODR : SPIRV::LinkageType::Export); Register Reg = GR.buildGlobalVariable(ResVReg, ResType, GlobalIdent, GV, Storage, Init, GlobalVar->isConstant(), HasLnkTy, LnkType, MIRBuilder, true); return Reg.isValid(); } bool SPIRVInstructionSelector::selectLog10(Register ResVReg, const SPIRVType *ResType, MachineInstr &I) const { if (STI.canUseExtInstSet(SPIRV::InstructionSet::OpenCL_std)) { return selectExtInst(ResVReg, ResType, I, CL::log10); } // There is no log10 instruction in the GLSL Extended Instruction set, so it // is implemented as: // log10(x) = log2(x) * (1 / log2(10)) // = log2(x) * 0.30103 MachineIRBuilder MIRBuilder(I); MachineBasicBlock &BB = *I.getParent(); // Build log2(x). Register VarReg = MRI->createVirtualRegister(&SPIRV::IDRegClass); bool Result = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpExtInst)) .addDef(VarReg) .addUse(GR.getSPIRVTypeID(ResType)) .addImm(static_cast(SPIRV::InstructionSet::GLSL_std_450)) .addImm(GL::Log2) .add(I.getOperand(1)) .constrainAllUses(TII, TRI, RBI); // Build 0.30103. assert(ResType->getOpcode() == SPIRV::OpTypeVector || ResType->getOpcode() == SPIRV::OpTypeFloat); // TODO: Add matrix implementation once supported by the HLSL frontend. const SPIRVType *SpirvScalarType = ResType->getOpcode() == SPIRV::OpTypeVector ? GR.getSPIRVTypeForVReg(ResType->getOperand(1).getReg()) : ResType; Register ScaleReg = GR.buildConstantFP(APFloat(0.30103f), MIRBuilder, SpirvScalarType); // Multiply log2(x) by 0.30103 to get log10(x) result. auto Opcode = ResType->getOpcode() == SPIRV::OpTypeVector ? SPIRV::OpVectorTimesScalar : SPIRV::OpFMulS; Result &= BuildMI(BB, I, I.getDebugLoc(), TII.get(Opcode)) .addDef(ResVReg) .addUse(GR.getSPIRVTypeID(ResType)) .addUse(VarReg) .addUse(ScaleReg) .constrainAllUses(TII, TRI, RBI); return Result; } bool SPIRVInstructionSelector::selectSpvThreadId(Register ResVReg, const SPIRVType *ResType, MachineInstr &I) const { // DX intrinsic: @llvm.dx.thread.id(i32) // ID Name Description // 93 ThreadId reads the thread ID MachineIRBuilder MIRBuilder(I); const SPIRVType *U32Type = GR.getOrCreateSPIRVIntegerType(32, MIRBuilder); const SPIRVType *Vec3Ty = GR.getOrCreateSPIRVVectorType(U32Type, 3, MIRBuilder); const SPIRVType *PtrType = GR.getOrCreateSPIRVPointerType( Vec3Ty, MIRBuilder, SPIRV::StorageClass::Input); // Create new register for GlobalInvocationID builtin variable. Register NewRegister = MIRBuilder.getMRI()->createVirtualRegister(&SPIRV::IDRegClass); MIRBuilder.getMRI()->setType(NewRegister, LLT::pointer(0, 32)); GR.assignSPIRVTypeToVReg(PtrType, NewRegister, MIRBuilder.getMF()); // Build GlobalInvocationID global variable with the necessary decorations. Register Variable = GR.buildGlobalVariable( NewRegister, PtrType, getLinkStringForBuiltIn(SPIRV::BuiltIn::GlobalInvocationId), nullptr, SPIRV::StorageClass::Input, nullptr, true, true, SPIRV::LinkageType::Import, MIRBuilder, false); // Create new register for loading value. MachineRegisterInfo *MRI = MIRBuilder.getMRI(); Register LoadedRegister = MRI->createVirtualRegister(&SPIRV::IDRegClass); MIRBuilder.getMRI()->setType(LoadedRegister, LLT::pointer(0, 32)); GR.assignSPIRVTypeToVReg(Vec3Ty, LoadedRegister, MIRBuilder.getMF()); // Load v3uint value from the global variable. BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(SPIRV::OpLoad)) .addDef(LoadedRegister) .addUse(GR.getSPIRVTypeID(Vec3Ty)) .addUse(Variable); // Get Thread ID index. Expecting operand is a constant immediate value, // wrapped in a type assignment. assert(I.getOperand(2).isReg()); Register ThreadIdReg = I.getOperand(2).getReg(); SPIRVType *ConstTy = this->MRI->getVRegDef(ThreadIdReg); assert(ConstTy && ConstTy->getOpcode() == SPIRV::ASSIGN_TYPE && ConstTy->getOperand(1).isReg()); Register ConstReg = ConstTy->getOperand(1).getReg(); const MachineInstr *Const = this->MRI->getVRegDef(ConstReg); assert(Const && Const->getOpcode() == TargetOpcode::G_CONSTANT); const llvm::APInt &Val = Const->getOperand(1).getCImm()->getValue(); const uint32_t ThreadId = Val.getZExtValue(); // Extract the thread ID from the loaded vector value. MachineBasicBlock &BB = *I.getParent(); auto MIB = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpCompositeExtract)) .addDef(ResVReg) .addUse(GR.getSPIRVTypeID(ResType)) .addUse(LoadedRegister) .addImm(ThreadId); return MIB.constrainAllUses(TII, TRI, RBI); } namespace llvm { InstructionSelector * createSPIRVInstructionSelector(const SPIRVTargetMachine &TM, const SPIRVSubtarget &Subtarget, const RegisterBankInfo &RBI) { return new SPIRVInstructionSelector(TM, Subtarget, RBI); } } // namespace llvm