Merge with upstream.

llvm/include/llvm/CodeGen/GlobalISel/CallLowering.h

@@ -306,14 +306,6 @@ class CallLowering {
                             unsigned MaxSizeBits = 0);
 
     virtual bool finalize(CCState &State) { return true; }
-
-    MachineIRBuilder &MIRBuilder;
-    MachineRegisterInfo &MRI;
-    CCAssignFn *AssignFn;
-
-  private:
-    bool IsIncomingArgumentHandler;
-    virtual void anchor();
   };
 
   /// Base class for ValueHandlers used for arguments coming into the current

llvm/include/llvm/CodeGen/GlobalISel/CombinerHelper.h

@@ -91,7 +91,7 @@ struct InstrImmPair {
 
 struct TypeImmPair {
   LLT Ty;
-  int64_t Imm;
+  uint64_t Imm;
 };
 
 using OperandBuildSteps =

llvm/include/llvm/CodeGen/GlobalISel/LegalizerHelper.h

@@ -186,7 +186,11 @@ class LegalizerHelper {
   void extractParts(Register Reg, LLT Ty, int NumParts,
                     SmallVectorImpl<Register> &VRegs);
 
-  /// Version which handles irregular splits.
+  /// Versions which handle irregular splits.
+  bool extractParts(Register Reg, LLT RegTy, LLT MainTy,
+                    LLT &LeftoverTy,
+                    SmallVectorImpl<Register> &VRegs,
+                    SmallVectorImpl<Register> &LeftoverVRegs);
   bool extractParts(Register Reg, LLT RegTy, LLT MainTy,
                     SmallVectorImpl<Register> &VRegs, LLT &LeftoverTy,
                     Register &LeftoverReg);

llvm/include/llvm/CodeGen/GlobalISel/LegalizerInfo.h

@@ -1155,7 +1155,8 @@ class LegalizerInfo {
   }
 
   virtual LegalizerHelper::LegalizeResult
-  legalizeCustomMaybeLegal(LegalizerHelper &Helper, MachineInstr &MI) const {
+  legalizeCustomMaybeLegal(LegalizerHelper &Helper, MachineInstr &MI,
+                           LostDebugLocObserver &LocObserver) const {
     return legalizeCustom(Helper, MI) ? LegalizerHelper::Legalized
                                       : LegalizerHelper::UnableToLegalize;
   }

llvm/include/llvm/Support/MachineValueType.h

@@ -1079,7 +1079,7 @@ namespace llvm {
     /// base size.
     TypeSize getStoreSize() const {
       TypeSize BaseSize = getSizeInBits();
-      return {(BaseSize.getKnownMinSize() + 7) / 8, BaseSize.isScalable()};
+      return {divideCeil(BaseSize.getKnownMinSize(), 8), BaseSize.isScalable()};
     }
 
     /// Return the number of bits overwritten by a store of the specified value

llvm/lib/CodeGen/GlobalISel/CallLowering.cpp

@@ -135,7 +135,7 @@ bool CallLowering::lowerCall(MachineIRBuilder &MIRBuilder, const CallBase &CB,
   else
     Info.Callee = MachineOperand::CreateReg(GetCalleeReg(), false);
 
-  Info.OrigRet = ArgInfo{ResRegs, CB, ISD::ArgFlagsTy{}};
+  Info.OrigRet = ArgInfo{ResRegs, CB, 0, ISD::ArgFlagsTy{}};
   if (!Info.OrigRet.Ty->isVoidTy())
     setArgFlags(Info.OrigRet, AttributeList::ReturnIndex, DL, CB);
 
@@ -504,7 +504,6 @@ static void buildCopyToRegs(MachineIRBuilder &B, ArrayRef<Register> DstRegs,
   LLT LCMTy = getLCMType(SrcTy, PartTy);
 
   const unsigned DstSize = DstTy.getSizeInBits();
-  const unsigned SrcSize = SrcTy.getSizeInBits();
   unsigned CoveringSize = LCMTy.getSizeInBits();
 
   Register UnmergeSrc = SrcReg;
@@ -611,10 +610,9 @@ bool CallLowering::determineAssignments(ValueAssigner &Assigner,
           Flags.setSplitEnd();
           if (!Exact && !CurVT.isVector())
             PartVT = TLI->getRegisterTypeForCallingConv(
-                F.getContext(), CCInfo.getCallingConv(),
-                EVT::getIntegerVT(F.getContext(),
-                                  CurVT.getSizeInBits() -
-                                      NewVT.getSizeInBits() * Part));
+                Ctx, CCInfo.getCallingConv(),
+                EVT::getIntegerVT(Ctx, CurVT.getSizeInBits() -
+                                           NewVT.getSizeInBits() * Part));
         }
       }
 
@@ -686,9 +684,11 @@ bool CallLowering::handleAssignments(ValueHandler &Handler,
         ArgReg = MRI.createGenericVirtualRegister(p0);
       else {
         MachineFrameInfo &MFI = MF.getFrameInfo();
-        unsigned Size = ValVT.getStoreSize();
-        int FI =
-            MFI.CreateStackObject(Size, DL.getPrefTypeAlign(Args[i].Ty), false);
+        // TODO: The memory size may be larger than the value we need to
+        // store. We may need to adjust the offset for big endian targets.
+        LLT MemTy = Handler.getStackValueStoreType(DL, VA, Args[i].Flags[0]);
+        int FI = MFI.CreateStackObject(MemTy.getSizeInBytes(),
+                                       DL.getPrefTypeAlign(Args[i].Ty), false);
         auto StackSlot = MIRBuilder.buildFrameIndex(p0, FI);
         LLT sIndex = LLT::scalar(DL.getIndexSizeInBits(0));
         ArgReg = {};
@@ -697,9 +697,9 @@ bool CallLowering::handleAssignments(ValueHandler &Handler,
         MachinePointerInfo MPO = MachinePointerInfo::getFixedStack(MF, FI);
         CCValAssign IndirectVA = CCValAssign::getMem(i, ValVT, IndirectOffset,
                                                      ValVT, CCValAssign::Full);
-        Handler.assignValueToAddress(Args[i].OrigRegs[0], ArgReg, Size, MPO,
+        Handler.assignValueToAddress(Args[i].OrigRegs[0], ArgReg, MemTy, MPO,
                                      IndirectVA);
-        IndirectOffset += Size;
+        IndirectOffset += MemTy.getSizeInBytes();
       }
       Args[i].Regs[0] = ArgReg;
     }
@@ -819,16 +819,18 @@ bool CallLowering::handleAssignments(ValueHandler &Handler,
 
       if (VA.getLocInfo() == CCValAssign::Indirect) {
         Register AddrReg;
-        unsigned Size = ValVT.getStoreSize();
+        // TODO: The memory size may be larger than the value we need to
+        // store. We may need to adjust the offset for big endian targets.
+        LLT MemTy = Handler.getStackValueStoreType(DL, VA, Args[i].Flags[0]);
         LLT sIndex = LLT::scalar(DL.getIndexSizeInBits(0));
         MIRBuilder.materializePtrAdd(AddrReg, Args[i].Regs[0], sIndex,
                                      IndirectOffset);
         MachinePointerInfo MPO(Args[i].OrigValue, IndirectOffset);
         CCValAssign IndirectVA =
           CCValAssign::getMem(i, ValVT, IndirectOffset, ValVT, CCValAssign::Full);
-        Handler.assignValueToAddress(Args[i].OrigRegs[0], AddrReg, Size, MPO,
+        Handler.assignValueToAddress(Args[i].OrigRegs[0], AddrReg, MemTy, MPO,
                                      IndirectVA);
-        IndirectOffset += Size;
+        IndirectOffset += MemTy.getSizeInBytes();
       }
     }
 

llvm/lib/CodeGen/GlobalISel/CombinerHelper.cpp

@@ -4546,7 +4546,7 @@ bool CombinerHelper::applyCombineOrToAdd(MachineInstr &MI) {
 bool CombinerHelper::matchCombineFunnelShift(MachineInstr &MI,
                                              FunnelShift &MatchInfo) {
   Register DstReg = MI.getOperand(0).getReg();
-  int64_t ShiftRightAmt;
+  uint64_t ShiftRightAmt;
   return mi_match(DstReg, MRI,
                   m_GAdd(m_GShl(m_Reg(MatchInfo.ShiftLeftReg),
                                 m_ICst(MatchInfo.ShiftLeftAmt)),

llvm/lib/CodeGen/GlobalISel/Legalizer.cpp

@@ -110,7 +110,6 @@ static bool isArtifact(const MachineInstr &MI) {
   case TargetOpcode::G_CONCAT_VECTORS:
   case TargetOpcode::G_BUILD_VECTOR:
   case TargetOpcode::G_EXTRACT:
-  case TargetOpcode::G_INSERT:
     return true;
   case TargetOpcode::G_INSERT:
     return AllowGInsertAsArtifact;

llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp

@@ -143,7 +143,7 @@ LegalizerHelper::legalizeInstrStep(MachineInstr &MI,
     return moreElementsVector(MI, Step.TypeIdx, Step.NewType);
   case Custom:
     LLVM_DEBUG(dbgs() << ".. Custom legalization\n");
-    return LI.legalizeCustomMaybeLegal(*this, MI);
+    return LI.legalizeCustomMaybeLegal(*this, MI, LocObserver);
   default:
     LLVM_DEBUG(dbgs() << ".. Unable to legalize\n");
     return UnableToLegalize;
@@ -162,6 +162,52 @@ void LegalizerHelper::extractParts(Register Reg, LLT Ty, int NumParts,
   MIRBuilder.buildUnmerge(VRegs, Reg);
 }
 
+bool LegalizerHelper::extractParts(Register Reg, LLT RegTy,
+                                   LLT MainTy, LLT &LeftoverTy,
+                                   SmallVectorImpl<Register> &VRegs,
+                                   SmallVectorImpl<Register> &LeftoverRegs) {
+  assert(!LeftoverTy.isValid() && "this is an out argument");
+
+  unsigned RegSize = RegTy.getSizeInBits();
+  unsigned MainSize = MainTy.getSizeInBits();
+  unsigned NumParts = RegSize / MainSize;
+  unsigned LeftoverSize = RegSize - NumParts * MainSize;
+
+  // Use an unmerge when possible.
+  if (LeftoverSize == 0) {
+    for (unsigned I = 0; I < NumParts; ++I)
+      VRegs.push_back(MRI.createGenericVirtualRegister(MainTy));
+    MIRBuilder.buildUnmerge(VRegs, Reg);
+    return true;
+  }
+
+  if (MainTy.isVector()) {
+    unsigned EltSize = MainTy.getScalarSizeInBits();
+    if (LeftoverSize % EltSize != 0)
+      return false;
+    LeftoverTy = LLT::scalarOrVector(
+        ElementCount::getFixed(LeftoverSize / EltSize), EltSize);
+  } else {
+    LeftoverTy = LLT::scalar(LeftoverSize);
+  }
+
+  // For irregular sizes, extract the individual parts.
+  for (unsigned I = 0; I != NumParts; ++I) {
+    Register NewReg = MRI.createGenericVirtualRegister(MainTy);
+    VRegs.push_back(NewReg);
+    MIRBuilder.buildExtract(NewReg, Reg, MainSize * I);
+  }
+
+  for (unsigned Offset = MainSize * NumParts; Offset < RegSize;
+       Offset += LeftoverSize) {
+    Register NewReg = MRI.createGenericVirtualRegister(LeftoverTy);
+    LeftoverRegs.push_back(NewReg);
+    MIRBuilder.buildExtract(NewReg, Reg, Offset);
+  }
+
+  return true;
+}
+
 bool LegalizerHelper::extractParts(Register Reg, LLT RegTy, LLT MainTy,
                                    SmallVectorImpl<Register> &VRegs,
                                    LLT &LeftoverTy, Register &LeftoverReg) {
@@ -795,8 +841,8 @@ LegalizerHelper::libcall(MachineInstr &MI, LostDebugLocObserver &LocObserver) {
     unsigned OpSize = MRI.getType(MI.getOperand(1).getReg()).getSizeInBits();
     Type *OpTy = IntegerType::get(Ctx, OpSize);
     RTLIB::Libcall Libcall = getRTLibDesc(MI.getOpcode(), OpSize);
-    createLibcall(MIRBuilder, Libcall, {MI.getOperand(0).getReg(), ResTy},
-                  {{MI.getOperand(1).getReg(), OpTy}});
+    createLibcall(MIRBuilder, Libcall, {MI.getOperand(0).getReg(), ResTy, 0},
+                  {{MI.getOperand(1).getReg(), OpTy, 0}});
     break;
   }
   case TargetOpcode::G_SHL:
@@ -807,8 +853,9 @@ LegalizerHelper::libcall(MachineInstr &MI, LostDebugLocObserver &LocObserver) {
     Register AmountReg = MI.getOperand(2).getReg();
     Type *AmountTy =
         IntegerType::get(Ctx, MRI.getType(AmountReg).getSizeInBits());
-    createLibcall(MIRBuilder, Libcall, {MI.getOperand(0).getReg(), OpTy},
-                  {{MI.getOperand(1).getReg(), OpTy}, {AmountReg, AmountTy}});
+    createLibcall(
+        MIRBuilder, Libcall, {MI.getOperand(0).getReg(), OpTy, 0},
+        {{MI.getOperand(1).getReg(), OpTy, 0}, {AmountReg, AmountTy, 0}});
     break;
   }
   case TargetOpcode::G_INTRINSIC_TRUNC:
@@ -4238,17 +4285,22 @@ LegalizerHelper::reduceLoadStoreWidth(GLoadStore &LdStMI, unsigned TypeIdx,
     return UnableToLegalize;
   }
 
-  unsigned NumParts = 0;
+  int NumParts = -1;
+  int NumLeftover = -1;
   LLT LeftoverTy;
-  SmallVector<Register, 8> NarrowRegs;
-  Register NarrowLeftoverReg;
+  SmallVector<Register, 8> NarrowRegs, NarrowLeftoverRegs;
   if (IsLoad) {
     NumParts = getNarrowTypeBreakDown(ValTy, NarrowTy, LeftoverTy);
-    NarrowRegs.resize(NumParts);
-  } else if (extractParts(ValReg, ValTy, NarrowTy, NarrowRegs, LeftoverTy,
-                        NarrowLeftoverReg))
-    NumParts = NarrowRegs.size();
-  if (!NumParts)
+    NumLeftover = LeftoverTy.isValid();
+  } else {
+    if (extractParts(ValReg, ValTy, NarrowTy, LeftoverTy, NarrowRegs,
+                     NarrowLeftoverRegs)) {
+      NumParts = NarrowRegs.size();
+      NumLeftover = NarrowLeftoverRegs.size();
+    }
+  }
+
+  if (NumParts == -1)
     return UnableToLegalize;
 
   LLT PtrTy = MRI.getType(AddrReg);
@@ -4272,16 +4324,16 @@ LegalizerHelper::reduceLoadStoreWidth(GLoadStore &LdStMI, unsigned TypeIdx,
 
       MIRBuilder.materializePtrAdd(NewAddrReg, AddrReg, OffsetTy, ByteOffset);
 
-      MachineMemOperand *NewMMO =
-          MF.getMachineMemOperand(&MMO, ByteOffset, PartTy);
+            MachineMemOperand *NewMMO =
+                MF.getMachineMemOperand(&MMO, ByteOffset, PartTy);
 
-      if (IsLoad) {
-        Register Dst = MRI.createGenericVirtualRegister(PartTy);
-        ValRegs[Idx] = Dst;
-        MIRBuilder.buildLoad(Dst, NewAddrReg, *NewMMO);
-      } else {
-        MIRBuilder.buildStore(ValRegs[Idx], NewAddrReg, *NewMMO);
-      }
+            if (IsLoad) {
+              Register Dst = MRI.createGenericVirtualRegister(PartTy);
+              ValRegs.push_back(Dst);
+              MIRBuilder.buildLoad(Dst, NewAddrReg, *NewMMO);
+            } else {
+              MIRBuilder.buildStore(ValRegs[Idx], NewAddrReg, *NewMMO);
+            }
     }
 
     return Offset;
@@ -4291,11 +4343,12 @@ LegalizerHelper::reduceLoadStoreWidth(GLoadStore &LdStMI, unsigned TypeIdx,
 
   // Handle the rest of the register if this isn't an even type breakdown.
   if (LeftoverTy.isValid())
-    splitTypePieces(LeftoverTy, NarrowLeftoverReg, HandledOffset);
+    splitTypePieces(LeftoverTy, NarrowLeftoverRegs, HandledOffset);
 
-  if (IsLoad)
+  if (IsLoad) {
     insertParts(ValReg, ValTy, NarrowTy, NarrowRegs, LeftoverTy,
-                NarrowLeftoverReg);
+                NarrowLeftoverRegs[0]);
+  }
 
   LdStMI.eraseFromParent();
   return Legalized;
@@ -5508,15 +5561,13 @@ LegalizerHelper::narrowScalarInsert(MachineInstr &MI, unsigned TypeIdx,
     return UnableToLegalize;
 
   unsigned DstSize = MRI.getType(MI.getOperand(0).getReg()).getSizeInBits();
-  unsigned NarrowSize = NarrowTy.getSizeInBits();
+  uint64_t NarrowSize = NarrowTy.getSizeInBits();
 
   // FIXME: add support for when DstSize isn't an exact multiple of
   // NarrowSize.
   if (DstSize % NarrowSize != 0)
     return UnableToLegalize;
 
-  int NumParts = DstSize / NarrowSize;
-
   SmallVector<Register, 2> SrcRegs, LeftoverRegs, DstRegs;
   SmallVector<uint64_t, 2> Indexes;
   LLT RegTy = MRI.getType(MI.getOperand(0).getReg());
@@ -5527,7 +5578,6 @@ LegalizerHelper::narrowScalarInsert(MachineInstr &MI, unsigned TypeIdx,
   for (Register Reg : LeftoverRegs)
     SrcRegs.push_back(Reg);
 
-  uint64_t NarrowSize = NarrowTy.getSizeInBits();
   Register OpReg = MI.getOperand(2).getReg();
   uint64_t OpStart = MI.getOperand(3).getImm();
   uint64_t OpSize = MRI.getType(OpReg).getSizeInBits();
@@ -5556,8 +5606,8 @@ LegalizerHelper::narrowScalarInsert(MachineInstr &MI, unsigned TypeIdx,
 
     // OpSegStart is where this destination segment would start in OpReg if it
     // extended infinitely in both directions.
-    unsigned ExtractOffset, InsertOffset;
-    unsigned SegSize;
+    int64_t ExtractOffset, InsertOffset;
+    uint64_t SegSize;
     if (OpStart < DstStart) {
       InsertOffset = 0;
       ExtractOffset = DstStart - OpStart;

llvm/lib/Target/AMDGPU/AMDGPUCallLowering.cpp

@@ -1230,7 +1230,7 @@ bool AMDGPUCallLowering::lowerTailCall(
   if (MIB->getOperand(0).isReg()) {
     MIB->getOperand(0).setReg(constrainOperandRegClass(
         MF, *TRI, MRI, *ST.getInstrInfo(), *ST.getRegBankInfo(), *MIB,
-        MIB->getDesc(), MIB->getOperand(0), 0));
+        MIB->getOperand(0), 0));
   }
 
   MF.getFrameInfo().setHasTailCall();