Merge pull request #20264 from rmnattas/oh-card

OffHeap CardMarking using the baseObj instead of dataAddr
eclipse-openj9 · Dec 11, 2024 · cdc1adc · cdc1adc
2 parents 20b8e7c + 17f2c1f
commit cdc1adc
Show file tree

Hide file tree

Showing 4 changed files with 73 additions and 9 deletions.
diff --git a/runtime/compiler/aarch64/codegen/J9TreeEvaluator.cpp b/runtime/compiler/aarch64/codegen/J9TreeEvaluator.cpp
@@ -987,7 +987,8 @@ VMCardCheckEvaluator(
       TR::Register *dstReg,
       TR_ARM64ScratchRegisterManager *srm,
       TR::LabelSymbol *doneLabel,
-      TR::CodeGenerator *cg)
+      TR::CodeGenerator *cg,
+      bool clobberDstReg=false)
    {
    TR::Compilation *comp = cg->comp();
 
@@ -1014,7 +1015,11 @@ VMCardCheckEvaluator(
       cg->generateDebugCounter(TR::DebugCounter::debugCounterName(comp, "wrtbarEvaluator:020VMCardCheckEvaluator:01markThreadActiveCheckDone"), *srm);
       }
 
-   TR::Register *temp2Reg = srm->findOrCreateScratchRegister();
+   TR::Register *temp2Reg;
+   if (clobberDstReg)
+      temp2Reg = dstReg;
+   else
+      temp2Reg = srm->findOrCreateScratchRegister();
       /*
        * Generating code checking whether an object is in heap
        *
@@ -5083,6 +5088,12 @@ static TR::Register *VMinlineCompareAndSwapObject(TR::Node *node, TR::CodeGenera
    oldVNode = node->getChild(3);
    newVNode = node->getChild(4);
 
+#if defined(J9VM_GC_SPARSE_HEAP_ALLOCATION)
+   bool isObjOffHeapDataAddr = (gcMode == gc_modron_wrtbar_cardmark_incremental && TR::Compiler->om.isOffHeapAllocationEnabled() && objNode->isDataAddrPointer());
+   TR::Register *baseObjReg = NULL;
+   if (isObjOffHeapDataAddr)
+      TR::TreeEvaluator::stopUsingCopyReg(objNode->getFirstChild(), baseObjReg, cg);
+#endif /* defined(J9VM_GC_SPARSE_HEAP_ALLOCATION) */
    objReg = cg->evaluate(objNode);
 
    if (offsetNode->getOpCode().isLoadConst() && offsetNode->getRegister() == NULL)
@@ -5253,11 +5264,20 @@ static TR::Register *VMinlineCompareAndSwapObject(TR::Node *node, TR::CodeGenera
             generateCompareBranchInstruction(cg, TR::InstOpCode::cbzx, node, wrtBarSrcReg, doneLabel);
             cg->generateDebugCounter(TR::DebugCounter::debugCounterName(comp, "wrtbarEvaluator:000srcNullChk:NonNull"), *srm);
             }
-         VMCardCheckEvaluator(node, objReg, srm, doneLabel, cg);
+#if defined(J9VM_GC_SPARSE_HEAP_ALLOCATION)
+         if (isObjOffHeapDataAddr)
+            VMCardCheckEvaluator(node, baseObjReg, srm, doneLabel, cg, true);
+         else
+#endif /* defined(J9VM_GC_SPARSE_HEAP_ALLOCATION) */
+            VMCardCheckEvaluator(node, objReg, srm, doneLabel, cg);
          }
 
       TR_ARM64ScratchRegisterDependencyConditions scratchDeps;
       scratchDeps.addDependency(cg, objReg, doWrtBar ? TR::RealRegister::x0 : TR::RealRegister::NoReg);
+#if defined(J9VM_GC_SPARSE_HEAP_ALLOCATION)
+      if (isObjOffHeapDataAddr)
+         scratchDeps.addDependency(cg, baseObjReg, TR::RealRegister::NoReg);
+#endif /* defined(J9VM_GC_SPARSE_HEAP_ALLOCATION) */
       scratchDeps.addDependency(cg, wrtBarSrcReg, doWrtBar ? TR::RealRegister::x1 : TR::RealRegister::NoReg);
       if (offsetInReg)
          {

diff --git a/runtime/compiler/p/codegen/J9TreeEvaluator.cpp b/runtime/compiler/p/codegen/J9TreeEvaluator.cpp
@@ -8292,6 +8292,12 @@ static TR::Register *VMinlineCompareAndSetOrExchangeReference(TR::Node *node, TR
    oldVNode = node->getChild(3);
    newVNode = node->getChild(4);
 
+#if defined(J9VM_GC_SPARSE_HEAP_ALLOCATION)
+   bool isObjOffHeapDataAddr = (gcMode == gc_modron_wrtbar_cardmark_incremental && TR::Compiler->om.isOffHeapAllocationEnabled() && objNode->isDataAddrPointer());
+   TR::Register *baseObjReg = NULL;
+   if (isObjOffHeapDataAddr)
+      TR::TreeEvaluator::stopUsingCopyReg(objNode->getFirstChild(), baseObjReg, cg);
+#endif /* defined(J9VM_GC_SPARSE_HEAP_ALLOCATION) */
    objReg = cg->evaluate(objNode);
 
    // VM helper chops off the value in 32bit, and we don't want the whole long value either
@@ -8619,7 +8625,13 @@ static TR::Register *VMinlineCompareAndSetOrExchangeReference(TR::Node *node, TR
       }
    else if (!doWrtBar && doCrdMrk)
       {
-      TR::Register *temp1Reg = cg->allocateRegister(), *temp2Reg = cg->allocateRegister(), *temp3Reg;
+      TR::Register *temp1Reg = cg->allocateRegister(), *temp2Reg, *temp3Reg;
+#if defined(J9VM_GC_SPARSE_HEAP_ALLOCATION)
+      if (isObjOffHeapDataAddr)
+         temp2Reg = baseObjReg; // Use baseObjReg as temp2Reg
+      else
+#endif /* defined(J9VM_GC_SPARSE_HEAP_ALLOCATION) */
+         temp2Reg = cg->allocateRegister();
       TR::addDependency(conditions, objReg, TR::RealRegister::NoReg, TR_GPR, cg);
       conditions->getPostConditions()->getRegisterDependency(0)->setExcludeGPR0();
       TR::addDependency(conditions, temp1Reg, TR::RealRegister::NoReg, TR_GPR, cg);
@@ -8652,7 +8664,13 @@ static TR::Register *VMinlineCompareAndSetOrExchangeReference(TR::Node *node, TR
          TR::addDependency(conditions, temp3Reg, TR::RealRegister::NoReg, TR_GPR, cg);
          }
 
-      VMCardCheckEvaluator(node, objReg, cndReg, temp1Reg, temp2Reg, temp3Reg, conditions, cg);
+#if defined(J9VM_GC_SPARSE_HEAP_ALLOCATION)
+      if (isObjOffHeapDataAddr)
+         // Given that baseObjReg is trash-able it can be used as the objReg and temp2Reg
+         VMCardCheckEvaluator(node, baseObjReg, cndReg, temp1Reg, temp2Reg, temp3Reg, conditions, cg);
+      else
+#endif /* defined(J9VM_GC_SPARSE_HEAP_ALLOCATION) */
+         VMCardCheckEvaluator(node, objReg, cndReg, temp1Reg, temp2Reg, temp3Reg, conditions, cg);
 
       cg->stopUsingRegister(temp1Reg);
       cg->stopUsingRegister(temp2Reg);

diff --git a/runtime/compiler/x/codegen/J9TreeEvaluator.cpp b/runtime/compiler/x/codegen/J9TreeEvaluator.cpp
@@ -11073,7 +11073,13 @@ void J9::X86::TreeEvaluator::VMwrtbarWithoutStoreEvaluator(
    TR::Register *owningObjectReg;
    TR::Register *tempReg = NULL;
 
-   owningObjectReg = cg->evaluate(destOwningObject);
+#if defined(J9VM_GC_SPARSE_HEAP_ALLOCATION)
+   bool stopUsingCopyBaseReg;
+   if (gcMode == gc_modron_wrtbar_cardmark_incremental && TR::Compiler->om.isOffHeapAllocationEnabled() && destOwningObject->isDataAddrPointer())
+      owningObjectReg = cg->evaluate(destOwningObject->getFirstChild());
+   else
+#endif /* defined(J9VM_GC_SPARSE_HEAP_ALLOCATION) */
+      owningObjectReg = cg->evaluate(destOwningObject);
 
    if (doInternalControlFlow)
       {

diff --git a/runtime/compiler/z/codegen/J9TreeEvaluator.cpp b/runtime/compiler/z/codegen/J9TreeEvaluator.cpp
@@ -12153,6 +12153,12 @@ J9::Z::TreeEvaluator::VMinlineCompareAndSwap(TR::Node *node, TR::CodeGenerator *
 
    // Eval old and new vals
    //
+#if defined(J9VM_GC_SPARSE_HEAP_ALLOCATION)
+   bool isObjOffHeapDataAddr = (TR::Compiler->om.writeBarrierType() == gc_modron_wrtbar_cardmark_incremental && TR::Compiler->om.isOffHeapAllocationEnabled() && objNode->isDataAddrPointer());
+   TR::Register *baseObjReg = NULL;
+   if (isObjOffHeapDataAddr)
+      baseObjReg = cg->gprClobberEvaluate(objNode->getFirstChild());
+#endif /* defined(J9VM_GC_SPARSE_HEAP_ALLOCATION) */
    objReg  = cg->evaluate(objNode);
    oldVReg = cg->gprClobberEvaluate(oldVNode);  //  CS oldReg, newReg, OFF(objReg)
    newVReg = cg->evaluate(newVNode);                    //    oldReg is clobbered
@@ -12285,15 +12291,24 @@ J9::Z::TreeEvaluator::VMinlineCompareAndSwap(TR::Node *node, TR::CodeGenerator *
    if (isObj && (doWrtBar || doCrdMrk))
       {
       TR::LabelSymbol *doneLabelWrtBar = generateLabelSymbol(cg);
-      TR::Register *epReg = cg->allocateRegister();
+      TR::Register *epReg = NULL;
+#if defined(J9VM_GC_SPARSE_HEAP_ALLOCATION)
+      if (!isObjOffHeapDataAddr)
+#endif /* defined(J9VM_GC_SPARSE_HEAP_ALLOCATION) */
+         epReg = cg->allocateRegister();
       TR::Register *raReg = cg->allocateRegister();
       TR::RegisterDependencyConditions* condWrtBar = new (cg->trHeapMemory()) TR::RegisterDependencyConditions(0, 5, cg);
       condWrtBar->addPostCondition(objReg, TR::RealRegister::GPR1);
       if (decompressedValueRegister != newVReg)
          condWrtBar->addPostCondition(newVReg, TR::RealRegister::AssignAny); //defect 92001
       if (decompressedValueRegister != objReg)  // add this because I got conflicting dependencies on GPR1 and GPR2!
          condWrtBar->addPostCondition(decompressedValueRegister, TR::RealRegister::GPR2); //defect 92001
-      condWrtBar->addPostCondition(epReg, cg->getEntryPointRegister());
+#if defined(J9VM_GC_SPARSE_HEAP_ALLOCATION)
+      if (isObjOffHeapDataAddr)
+         condWrtBar->addPostCondition(baseObjReg, cg->getEntryPointRegister());
+      else
+#endif /* defined(J9VM_GC_SPARSE_HEAP_ALLOCATION) */
+         condWrtBar->addPostCondition(epReg, cg->getEntryPointRegister());
       condWrtBar->addPostCondition(raReg, cg->getReturnAddressRegister());
       // Cardmarking is not inlined for gencon. Consider doing so when perf issue arises.
       if (doWrtBar)
@@ -12309,7 +12324,12 @@ J9::Z::TreeEvaluator::VMinlineCompareAndSwap(TR::Node *node, TR::CodeGenerator *
          }
       else if (doCrdMrk)
          {
-         VMCardCheckEvaluator(node, objReg, epReg, condWrtBar, cg, false, doneLabelWrtBar, false);
+#if defined(J9VM_GC_SPARSE_HEAP_ALLOCATION)
+         if (isObjOffHeapDataAddr)
+            VMCardCheckEvaluator(node, baseObjReg, NULL, condWrtBar, cg, true, doneLabelWrtBar, false);
+         else
+#endif /* defined(J9VM_GC_SPARSE_HEAP_ALLOCATION) */
+            VMCardCheckEvaluator(node, objReg, epReg, condWrtBar, cg, false, doneLabelWrtBar, false);
                                                                            // true #1 -> copy of objReg just happened, it's safe to clobber tempReg
                                                                            // false #2 -> Don't do compile time check for heap obj
          }