From a89a8e5b523d868e7e4244a41a5e41ac906fbbe6 Mon Sep 17 00:00:00 2001 From: Abdulrahman Alattas Date: Fri, 27 Sep 2024 15:15:44 -0400 Subject: [PATCH 1/4] Use baseObj for Unsafe.CAS CardMarking for OffHeap on P When evaluating Unsafe.CAS while running with Balanced GC and OffHeap enabled, if the object child is the dataAddrPointer load, pass the baseObj to VMCardCheckEvaluator for correct card marking. The dstReg and temp2Reg in VMCardCheckEvaluator can share a reg. Signed-off-by: Abdulrahman Alattas --- .../compiler/p/codegen/J9TreeEvaluator.cpp | 22 +++++++++++++++++-- 1 file changed, 20 insertions(+), 2 deletions(-) diff --git a/runtime/compiler/p/codegen/J9TreeEvaluator.cpp b/runtime/compiler/p/codegen/J9TreeEvaluator.cpp index cf7f96a1fd1..b0e8c1d91a3 100644 --- a/runtime/compiler/p/codegen/J9TreeEvaluator.cpp +++ b/runtime/compiler/p/codegen/J9TreeEvaluator.cpp @@ -8292,6 +8292,12 @@ static TR::Register *VMinlineCompareAndSetOrExchangeReference(TR::Node *node, TR oldVNode = node->getChild(3); newVNode = node->getChild(4); +#if defined(J9VM_GC_SPARSE_HEAP_ALLOCATION) + bool isObjOffHeapDataAddr = (gcMode == gc_modron_wrtbar_cardmark_incremental && TR::Compiler->om.isOffHeapAllocationEnabled() && objNode->isDataAddrPointer()); + TR::Register *baseObjReg = NULL; + if (isObjOffHeapDataAddr) + TR::TreeEvaluator::stopUsingCopyReg(objNode->getFirstChild(), baseObjReg, cg); +#endif /* defined(J9VM_GC_SPARSE_HEAP_ALLOCATION) */ objReg = cg->evaluate(objNode); // VM helper chops off the value in 32bit, and we don't want the whole long value either @@ -8619,7 +8625,13 @@ static TR::Register *VMinlineCompareAndSetOrExchangeReference(TR::Node *node, TR } else if (!doWrtBar && doCrdMrk) { - TR::Register *temp1Reg = cg->allocateRegister(), *temp2Reg = cg->allocateRegister(), *temp3Reg; + TR::Register *temp1Reg = cg->allocateRegister(), *temp2Reg, *temp3Reg; +#if defined(J9VM_GC_SPARSE_HEAP_ALLOCATION) + if (isObjOffHeapDataAddr) + temp2Reg = baseObjReg; // Use baseObjReg as temp2Reg + else +#endif /* defined(J9VM_GC_SPARSE_HEAP_ALLOCATION) */ + temp2Reg = cg->allocateRegister(); TR::addDependency(conditions, objReg, TR::RealRegister::NoReg, TR_GPR, cg); conditions->getPostConditions()->getRegisterDependency(0)->setExcludeGPR0(); TR::addDependency(conditions, temp1Reg, TR::RealRegister::NoReg, TR_GPR, cg); @@ -8652,7 +8664,13 @@ static TR::Register *VMinlineCompareAndSetOrExchangeReference(TR::Node *node, TR TR::addDependency(conditions, temp3Reg, TR::RealRegister::NoReg, TR_GPR, cg); } - VMCardCheckEvaluator(node, objReg, cndReg, temp1Reg, temp2Reg, temp3Reg, conditions, cg); +#if defined(J9VM_GC_SPARSE_HEAP_ALLOCATION) + if (isObjOffHeapDataAddr) + // Given that baseObjReg is trash-able it can be used as the objReg and temp2Reg + VMCardCheckEvaluator(node, baseObjReg, cndReg, temp1Reg, temp2Reg, temp3Reg, conditions, cg); + else +#endif /* defined(J9VM_GC_SPARSE_HEAP_ALLOCATION) */ + VMCardCheckEvaluator(node, objReg, cndReg, temp1Reg, temp2Reg, temp3Reg, conditions, cg); cg->stopUsingRegister(temp1Reg); cg->stopUsingRegister(temp2Reg); From 4f67854a39ce2a9c72b341725e3ba0a0ebeda8b5 Mon Sep 17 00:00:00 2001 From: Abdulrahman Alattas Date: Mon, 30 Sep 2024 12:16:58 -0400 Subject: [PATCH 2/4] Use baseObj in CardMarking for OffHeap on X When running with Balanced GC and OffHeap enabled, if the destOwningObject is a dataAddrPointer load, use the baseObj as the owningObjectReg. Signed-off-by: Abdulrahman Alattas --- runtime/compiler/x/codegen/J9TreeEvaluator.cpp | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/runtime/compiler/x/codegen/J9TreeEvaluator.cpp b/runtime/compiler/x/codegen/J9TreeEvaluator.cpp index 7b368f4d2e1..2cde918d260 100644 --- a/runtime/compiler/x/codegen/J9TreeEvaluator.cpp +++ b/runtime/compiler/x/codegen/J9TreeEvaluator.cpp @@ -11073,7 +11073,13 @@ void J9::X86::TreeEvaluator::VMwrtbarWithoutStoreEvaluator( TR::Register *owningObjectReg; TR::Register *tempReg = NULL; - owningObjectReg = cg->evaluate(destOwningObject); +#if defined(J9VM_GC_SPARSE_HEAP_ALLOCATION) + bool stopUsingCopyBaseReg; + if (gcMode == gc_modron_wrtbar_cardmark_incremental && TR::Compiler->om.isOffHeapAllocationEnabled() && destOwningObject->isDataAddrPointer()) + owningObjectReg = cg->evaluate(destOwningObject->getFirstChild()); + else +#endif /* defined(J9VM_GC_SPARSE_HEAP_ALLOCATION) */ + owningObjectReg = cg->evaluate(destOwningObject); if (doInternalControlFlow) { From 198c1611f409c8ee4b9bdf226eb02d8e9c5eb2f1 Mon Sep 17 00:00:00 2001 From: Abdulrahman Alattas Date: Mon, 30 Sep 2024 13:07:30 -0400 Subject: [PATCH 3/4] Use baseObj for Unsafe.CAS CardMarking for OffHeap on AArch64 When evaluating Unsafe.CAS while running with Balanced GC and OffHeap enabled, if the object child is the dataAddrPointer load, pass the baseObj to VMCardCheckEvaluator for correct card marking. The dstReg and temp2Reg in VMCardCheckEvaluator can share a reg, adding the argument clobberDstReg to indicate if dstReg can be used as the temp2Reg. Not allocating temp2Reg and adding a dep for baseObjReg uses the same number of registers. Signed-off-by: Abdulrahman Alattas --- .../aarch64/codegen/J9TreeEvaluator.cpp | 26 ++++++++++++++++--- 1 file changed, 23 insertions(+), 3 deletions(-) diff --git a/runtime/compiler/aarch64/codegen/J9TreeEvaluator.cpp b/runtime/compiler/aarch64/codegen/J9TreeEvaluator.cpp index f12e6f3ea21..d3891f194b8 100644 --- a/runtime/compiler/aarch64/codegen/J9TreeEvaluator.cpp +++ b/runtime/compiler/aarch64/codegen/J9TreeEvaluator.cpp @@ -987,7 +987,8 @@ VMCardCheckEvaluator( TR::Register *dstReg, TR_ARM64ScratchRegisterManager *srm, TR::LabelSymbol *doneLabel, - TR::CodeGenerator *cg) + TR::CodeGenerator *cg, + bool clobberDstReg=false) { TR::Compilation *comp = cg->comp(); @@ -1014,7 +1015,11 @@ VMCardCheckEvaluator( cg->generateDebugCounter(TR::DebugCounter::debugCounterName(comp, "wrtbarEvaluator:020VMCardCheckEvaluator:01markThreadActiveCheckDone"), *srm); } - TR::Register *temp2Reg = srm->findOrCreateScratchRegister(); + TR::Register *temp2Reg; + if (clobberDstReg) + temp2Reg = dstReg; + else + temp2Reg = srm->findOrCreateScratchRegister(); /* * Generating code checking whether an object is in heap * @@ -5083,6 +5088,12 @@ static TR::Register *VMinlineCompareAndSwapObject(TR::Node *node, TR::CodeGenera oldVNode = node->getChild(3); newVNode = node->getChild(4); +#if defined(J9VM_GC_SPARSE_HEAP_ALLOCATION) + bool isObjOffHeapDataAddr = (gcMode == gc_modron_wrtbar_cardmark_incremental && TR::Compiler->om.isOffHeapAllocationEnabled() && objNode->isDataAddrPointer()); + TR::Register *baseObjReg = NULL; + if (isObjOffHeapDataAddr) + TR::TreeEvaluator::stopUsingCopyReg(objNode->getFirstChild(), baseObjReg, cg); +#endif /* defined(J9VM_GC_SPARSE_HEAP_ALLOCATION) */ objReg = cg->evaluate(objNode); if (offsetNode->getOpCode().isLoadConst() && offsetNode->getRegister() == NULL) @@ -5253,11 +5264,20 @@ static TR::Register *VMinlineCompareAndSwapObject(TR::Node *node, TR::CodeGenera generateCompareBranchInstruction(cg, TR::InstOpCode::cbzx, node, wrtBarSrcReg, doneLabel); cg->generateDebugCounter(TR::DebugCounter::debugCounterName(comp, "wrtbarEvaluator:000srcNullChk:NonNull"), *srm); } - VMCardCheckEvaluator(node, objReg, srm, doneLabel, cg); +#if defined(J9VM_GC_SPARSE_HEAP_ALLOCATION) + if (isObjOffHeapDataAddr) + VMCardCheckEvaluator(node, baseObjReg, srm, doneLabel, cg, true); + else +#endif /* defined(J9VM_GC_SPARSE_HEAP_ALLOCATION) */ + VMCardCheckEvaluator(node, objReg, srm, doneLabel, cg); } TR_ARM64ScratchRegisterDependencyConditions scratchDeps; scratchDeps.addDependency(cg, objReg, doWrtBar ? TR::RealRegister::x0 : TR::RealRegister::NoReg); +#if defined(J9VM_GC_SPARSE_HEAP_ALLOCATION) + if (isObjOffHeapDataAddr) + scratchDeps.addDependency(cg, baseObjReg, TR::RealRegister::NoReg); +#endif /* defined(J9VM_GC_SPARSE_HEAP_ALLOCATION) */ scratchDeps.addDependency(cg, wrtBarSrcReg, doWrtBar ? TR::RealRegister::x1 : TR::RealRegister::NoReg); if (offsetInReg) { From 17f2c1f3a742ae5abeec5448fc6ada17c7566b67 Mon Sep 17 00:00:00 2001 From: Abdulrahman Alattas Date: Mon, 30 Sep 2024 14:06:38 -0400 Subject: [PATCH 4/4] Use baseObj for Unsafe.CAS CardMarking for OffHeap on Z When evaluating Unsafe.CAS while running with Balanced GC and OffHeap enabled, if the object child is the dataAddrPointer load, pass the baseObj to VMCardCheckEvaluator for correct card marking. Using the VMCardCheckEvaluator ability to clobberDstReg, we use baseObjReg instead of the temp epReg. Signed-off-by: Abdulrahman Alattas --- .../compiler/z/codegen/J9TreeEvaluator.cpp | 26 ++++++++++++++++--- 1 file changed, 23 insertions(+), 3 deletions(-) diff --git a/runtime/compiler/z/codegen/J9TreeEvaluator.cpp b/runtime/compiler/z/codegen/J9TreeEvaluator.cpp index 99d02b60f3b..fea14bd3221 100644 --- a/runtime/compiler/z/codegen/J9TreeEvaluator.cpp +++ b/runtime/compiler/z/codegen/J9TreeEvaluator.cpp @@ -12153,6 +12153,12 @@ J9::Z::TreeEvaluator::VMinlineCompareAndSwap(TR::Node *node, TR::CodeGenerator * // Eval old and new vals // +#if defined(J9VM_GC_SPARSE_HEAP_ALLOCATION) + bool isObjOffHeapDataAddr = (TR::Compiler->om.writeBarrierType() == gc_modron_wrtbar_cardmark_incremental && TR::Compiler->om.isOffHeapAllocationEnabled() && objNode->isDataAddrPointer()); + TR::Register *baseObjReg = NULL; + if (isObjOffHeapDataAddr) + baseObjReg = cg->gprClobberEvaluate(objNode->getFirstChild()); +#endif /* defined(J9VM_GC_SPARSE_HEAP_ALLOCATION) */ objReg = cg->evaluate(objNode); oldVReg = cg->gprClobberEvaluate(oldVNode); // CS oldReg, newReg, OFF(objReg) newVReg = cg->evaluate(newVNode); // oldReg is clobbered @@ -12285,7 +12291,11 @@ J9::Z::TreeEvaluator::VMinlineCompareAndSwap(TR::Node *node, TR::CodeGenerator * if (isObj && (doWrtBar || doCrdMrk)) { TR::LabelSymbol *doneLabelWrtBar = generateLabelSymbol(cg); - TR::Register *epReg = cg->allocateRegister(); + TR::Register *epReg = NULL; +#if defined(J9VM_GC_SPARSE_HEAP_ALLOCATION) + if (!isObjOffHeapDataAddr) +#endif /* defined(J9VM_GC_SPARSE_HEAP_ALLOCATION) */ + epReg = cg->allocateRegister(); TR::Register *raReg = cg->allocateRegister(); TR::RegisterDependencyConditions* condWrtBar = new (cg->trHeapMemory()) TR::RegisterDependencyConditions(0, 5, cg); condWrtBar->addPostCondition(objReg, TR::RealRegister::GPR1); @@ -12293,7 +12303,12 @@ J9::Z::TreeEvaluator::VMinlineCompareAndSwap(TR::Node *node, TR::CodeGenerator * condWrtBar->addPostCondition(newVReg, TR::RealRegister::AssignAny); //defect 92001 if (decompressedValueRegister != objReg) // add this because I got conflicting dependencies on GPR1 and GPR2! condWrtBar->addPostCondition(decompressedValueRegister, TR::RealRegister::GPR2); //defect 92001 - condWrtBar->addPostCondition(epReg, cg->getEntryPointRegister()); +#if defined(J9VM_GC_SPARSE_HEAP_ALLOCATION) + if (isObjOffHeapDataAddr) + condWrtBar->addPostCondition(baseObjReg, cg->getEntryPointRegister()); + else +#endif /* defined(J9VM_GC_SPARSE_HEAP_ALLOCATION) */ + condWrtBar->addPostCondition(epReg, cg->getEntryPointRegister()); condWrtBar->addPostCondition(raReg, cg->getReturnAddressRegister()); // Cardmarking is not inlined for gencon. Consider doing so when perf issue arises. if (doWrtBar) @@ -12309,7 +12324,12 @@ J9::Z::TreeEvaluator::VMinlineCompareAndSwap(TR::Node *node, TR::CodeGenerator * } else if (doCrdMrk) { - VMCardCheckEvaluator(node, objReg, epReg, condWrtBar, cg, false, doneLabelWrtBar, false); +#if defined(J9VM_GC_SPARSE_HEAP_ALLOCATION) + if (isObjOffHeapDataAddr) + VMCardCheckEvaluator(node, baseObjReg, NULL, condWrtBar, cg, true, doneLabelWrtBar, false); + else +#endif /* defined(J9VM_GC_SPARSE_HEAP_ALLOCATION) */ + VMCardCheckEvaluator(node, objReg, epReg, condWrtBar, cg, false, doneLabelWrtBar, false); // true #1 -> copy of objReg just happened, it's safe to clobber tempReg // false #2 -> Don't do compile time check for heap obj }