From a5105d629fd127c8ae16895522d73e74b051acb4 Mon Sep 17 00:00:00 2001
From: Theresa Mammarella <Theresa.T.Mammarella@ibm.com>
Date: Mon, 7 Oct 2024 15:53:00 -0400
Subject: [PATCH] Nullable array class cannot be cast to null-restricted

Signed-off-by: Theresa Mammarella <Theresa.T.Mammarella@ibm.com>
---
 runtime/oti/VMHelpers.hpp                     | 21 +++++++++++++------
 .../openj9/test/lworld/ValueTypeTests.java    | 11 ++++++++++
 2 files changed, 26 insertions(+), 6 deletions(-)

diff --git a/runtime/oti/VMHelpers.hpp b/runtime/oti/VMHelpers.hpp
index 6f6d368e99c..c3cab71cd0f 100644
--- a/runtime/oti/VMHelpers.hpp
+++ b/runtime/oti/VMHelpers.hpp
@@ -648,13 +648,22 @@ class VM_VMHelpers
 							/* check the [[O -> [[O case.  Don't allow [[I -> [[O */
 							if (instanceArity == castArity) {
 								J9Class *instanceClassLeafComponent = ((J9ArrayClass*)instanceClass)->leafComponentType;
-								if (J9CLASS_IS_MIXED(instanceClassLeafComponent)) {
-									/* we know arities are the same, so skip directly to the terminal case */
-									instanceClass = instanceClassLeafComponent;
-									castClass = castClassLeafComponent;
-									didRetry = true;
-									goto retry;
+#if defined(J9VM_OPT_VALHALLA_FLATTENABLE_VALUE_TYPES)
+								if (J9_IS_J9ARRAYCLASS_NULL_RESTRICTED(instanceClass)
+									|| !J9_IS_J9ARRAYCLASS_NULL_RESTRICTED(castClass)
+								) {
+#endif /* defined(J9VM_OPT_VALHALLA_FLATTENABLE_VALUE_TYPES) */
+									if (J9CLASS_IS_MIXED(instanceClassLeafComponent)) {
+										/* we know arities are the same, so skip directly to the terminal case */
+										instanceClass = instanceClassLeafComponent;
+										castClass = castClassLeafComponent;
+										didRetry = true;
+										goto retry;
+									}
+#if defined(J9VM_OPT_VALHALLA_FLATTENABLE_VALUE_TYPES)
 								}
+								/* else fail since a nullable array class cannot be cast to a null-restricted class */
+#endif /* defined(J9VM_OPT_VALHALLA_FLATTENABLE_VALUE_TYPES) */
 							}
 							/* else the arity of the instance wasn't high enough, so we fail */
 						}
diff --git a/test/functional/Valhalla/src_qtypes/org/openj9/test/lworld/ValueTypeTests.java b/test/functional/Valhalla/src_qtypes/org/openj9/test/lworld/ValueTypeTests.java
index 8952acd6508..c4eceb8e531 100644
--- a/test/functional/Valhalla/src_qtypes/org/openj9/test/lworld/ValueTypeTests.java
+++ b/test/functional/Valhalla/src_qtypes/org/openj9/test/lworld/ValueTypeTests.java
@@ -2291,6 +2291,17 @@ static public void testCheckCastNullableTypeOnNull() throws Throwable {
 		checkCastRefClassOnNull.invoke();
 	}
 
+	@Test(priority=1)
+	static public void testClassIsInstanceNullableArrays() throws Throwable {
+		ValueTypePoint2D[] nonNullableArray = (ValueTypePoint2D[]) ValueClass.newNullRestrictedArray(ValueTypePoint2D.class, 1);
+		ValueTypePoint2D[] nullableArray = new ValueTypePoint2D[1];
+
+		assertTrue(nonNullableArray.getClass().isInstance(nonNullableArray));
+		assertTrue(nullableArray.getClass().isInstance(nullableArray));
+		assertFalse(nonNullableArray.getClass().isInstance(nullableArray));
+		assertTrue(nullableArray.getClass().isInstance(nonNullableArray));
+	}
+
 	/*
 	 * Maintain a buffer of flattened arrays with long-aligned valuetypes while keeping a certain amount of classes alive at any
 	 * single time. This forces the GC to unload the classes.