Add JsonNodeFeature.FAIL_ON_NAN_TO_BIG_DECIMAL_COERCION to determine what happens on JsonNode coercion to BigDecimal with NaN

src/main/java/com/fasterxml/jackson/databind/cfg/JsonNodeFeature.java

@@ -61,8 +61,18 @@ public enum JsonNodeFeature implements DatatypeFeature
      *
      * @since 2.15
      */
-    STRIP_TRAILING_BIGDECIMAL_ZEROES(true)
-    ;
+    STRIP_TRAILING_BIGDECIMAL_ZEROES(true),
+
+    /**
+     *
+     * Feature that determines whether to fail on attempt to coercing {@code NaN} into {@link java.math.BigDecimal}
+     * because {@link com.fasterxml.jackson.databind.DeserializationFeature#USE_BIG_DECIMAL_FOR_FLOATS} is enabled.
+     *<p>
+     * Default value is {@code true} for backwards-compatibility, but will be changed to {@code false} in 3.0.
+     *
+     * @since 2.16
+     */
+    ALLOW_NaN_TO_BIG_DECIMAL_COERCION(true);
 
     private final static int FEATURE_INDEX = DatatypeFeatures.FEATURE_INDEX_JSON_NODE;
 

src/main/java/com/fasterxml/jackson/databind/deser/std/JsonNodeDeserializer.java

@@ -755,6 +755,12 @@ protected final JsonNode _fromFloat(JsonParser p, DeserializationContext ctxt,
             return _fromBigDecimal(ctxt, nodeFactory, p.getDecimalValue());
         }
         if (ctxt.isEnabled(DeserializationFeature.USE_BIG_DECIMAL_FOR_FLOATS)) {
+            // [databind#4194] Add an option to fail coercing NaN to BigDecimal
+            // Currently, Jackson 2.x allows such coercion, but Jackson 3.x will not
+            if (p.isNaN() && !ctxt.isEnabled(JsonNodeFeature.ALLOW_NaN_TO_BIG_DECIMAL_COERCION)) {
+                ctxt.handleWeirdNumberValue(handledType(), p.getDoubleValue(),
+                        "Cannot convert NaN into BigDecimal");
+            }
             try {
                 return _fromBigDecimal(ctxt, nodeFactory, p.getDecimalValue());
             } catch (NumberFormatException nfe) {

src/test/java/com/fasterxml/jackson/databind/node/NumberNodes1770Test.java

@@ -1,5 +1,7 @@
 package com.fasterxml.jackson.databind.node;
 
+import com.fasterxml.jackson.databind.cfg.JsonNodeFeature;
+import com.fasterxml.jackson.databind.exc.InvalidFormatException;
 import java.math.BigDecimal;
 
 import com.fasterxml.jackson.core.JsonFactory;
@@ -39,4 +41,36 @@ public void testBigDecimalCoercionInf() throws Exception
         assertTrue("Expected DoubleNode, got: "+jsonNode.getClass().getName()+": "+jsonNode, jsonNode.isDouble());
         assertEquals(Double.POSITIVE_INFINITY, jsonNode.doubleValue());
     }
+
+    // [databind#4194]: should be able to, by configuration, fail coercing NaN to BigDecimal
+    public void testBigDecimalCoercionNaN() throws Exception
+    {
+        _tryBigDecimalCoercionNaNWithOption(true);
+
+        try {
+            _tryBigDecimalCoercionNaNWithOption(false);
+            fail("Should not pass");
+        } catch (InvalidFormatException e) {
+            verifyException(e, "Cannot convert NaN");
+        }
+    }
+
+    private void _tryBigDecimalCoercionNaNWithOption(boolean isEnabled) throws Exception
+    {
+        JsonFactory factory = JsonFactory.builder()
+                .enable(JsonReadFeature.ALLOW_NON_NUMERIC_NUMBERS)
+                .build();
+        final ObjectReader reader = new JsonMapper(factory)
+                .reader()
+                .with(DeserializationFeature.USE_BIG_DECIMAL_FOR_FLOATS);
+
+        final String value = "NaN";
+        // depending on option
+        final JsonNode jsonNode = isEnabled
+                ? reader.with(JsonNodeFeature.ALLOW_NaN_TO_BIG_DECIMAL_COERCION).readTree(value)
+                : reader.without(JsonNodeFeature.ALLOW_NaN_TO_BIG_DECIMAL_COERCION).readTree(value);
+
+        assertTrue("Expected DoubleNode, got: "+jsonNode.getClass().getName()+": "+jsonNode, jsonNode.isDouble());
+        assertEquals(Double.NaN, jsonNode.doubleValue());
+    }
 }