Skip to content

Commit

Permalink
feat(JSON): accept int_lit & float_lit numbers as defined in go spec
Browse files Browse the repository at this point in the history 
The following numbers are now accepted by JSON, SuperJSONOf and
SubJSONOf operators:
    +42          → 42
    4_2          → 42
    0b101010     → 42
    0b10_1010    → 42
    0600         → 384
    0_600        → 384
    0o600        → 384
    0O600        → 384 // second character is capital letter 'O'
    0xBadFace    → 195951310
    0x_Bad_Face  → 195951310
    .25          → 0.25
    1_5.         → 15.0
    0.15e+0_2    → 15.0
    0x1p-2       → 0.25
    0x2.p10      → 2048.0
    0x1.Fp+0     → 1.9375
    0X.8p-0      → 0.5
    0X_1FFFP-16  → 0.1249847412109375

Signed-off-by: Maxime Soulé <[email protected]>
  • Loading branch information
@maxatome
maxatome committed Jan 5, 2022
1 parent 4e544bd commit 4aa5af6
Show file tree
Hide file tree
Showing 4 changed files with 198 additions and 35 deletions.
73 changes: 63 additions & 10 deletions internal/json/lex.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@ import (
"bytes"
"errors"
"fmt"
"math/big"
"strconv"
"strings"
"unicode"
Expand Down Expand Up @@ -198,7 +199,8 @@ func (j *json) nextToken(lval *yySymType) int {
return FALSE
}

case '-', '0', '1', '2', '3', '4', '5', '6', '7', '8', '9':
case '-', '0', '1', '2', '3', '4', '5', '6', '7', '8', '9',
'+', '.': // '+' & '.' are not normally accepted by JSON spec
n, ok := j.parseNumber()
if !ok {
return 0
Expand Down Expand Up @@ -337,21 +339,72 @@ str:
return "", false
}

const (
numInt = 1 << iota
numFloat
numGoExt
)

var numBytes = [...]uint8{
'+': numInt, '-': numInt,
'0': numInt,
'1': numInt,
'2': numInt,
'3': numInt,
'4': numInt,
'5': numInt,
'6': numInt,
'7': numInt,
'8': numInt,
'9': numInt,
'_': numGoExt,
// bases 2, 8, 16
'b': numInt, 'B': numInt, 'o': numInt, 'O': numInt, 'x': numInt, 'X': numInt,
'a': numInt, 'A': numInt,
'c': numInt, 'C': numInt,
'd': numInt, 'D': numInt,
'e': numInt | numFloat, 'E': numInt | numFloat,
'f': numInt, 'F': numInt,
// floats
'.': numFloat, 'p': numFloat, 'P': numFloat,
}

func (j *json) parseNumber() (float64, bool) {
// j.buf[j.pos.bpos] == '[0-9]' → caller responsibility
// j.buf[j.pos.bpos] == '[-+0-9.]' → caller responsibility

numKind := numBytes[j.buf[j.pos.bpos]]
i := j.pos.bpos + 1
l := len(j.buf)
num:
for ; i < l; i++ {
switch j.buf[i] {
case '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', '.', 'e', 'E', '+', '-':
default:
break num
for l := len(j.buf); i < l; i++ {
b := int(j.buf[i])
if b >= len(numBytes) || numBytes[b] == 0 {
break
}
numKind |= numBytes[b]
}

s := string(j.buf[j.pos.bpos:i])

var (
f float64
err error
)
// Differentiate float/int parsing to accept old octal notation:
// 0600 → 384 as int64, but 600 as float64
if (numKind & numFloat) != 0 {
// strconv.ParseFloat does not handle "_"
var bf *big.Float
bf, _, err = new(big.Float).Parse(s, 0)
if err == nil {
f, _ = bf.Float64()
}
} else { // numInt and/or numGoExt
var int int64
int, err = strconv.ParseInt(s, 0, 64)
if err == nil {
f = float64(int)
}
}

f, err := strconv.ParseFloat(string(j.buf[j.pos.bpos:i]), 64)
if err != nil {
j.fatal("invalid number")
return 0, false
Expand Down
75 changes: 75 additions & 0 deletions internal/json/parser_go113_test.go
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,75 @@
// Copyright (c) 2022, Maxime Soulé
// All rights reserved.
//
// This source code is licensed under the BSD-style license found in the
// LICENSE file in the root directory of this source tree.

//go:build go1.13
// +build go1.13

package json_test

import (
"testing"
)

func TestJSON_go113(t *testing.T) {
// Extend to golang 1.13 accepted numbers

// as int64
checkJSON(t, `4_2`, `42`)
checkJSON(t, `+4_2`, `42`)
checkJSON(t, `-4_2`, `-42`)

checkJSON(t, `0b101010`, `42`)
checkJSON(t, `-0b101010`, `-42`)
checkJSON(t, `+0b101010`, `42`)

checkJSON(t, `0b10_1010`, `42`)
checkJSON(t, `-0b_10_1010`, `-42`)
checkJSON(t, `+0b10_10_10`, `42`)

checkJSON(t, `0B101010`, `42`)
checkJSON(t, `-0B101010`, `-42`)
checkJSON(t, `+0B101010`, `42`)

checkJSON(t, `0B10_1010`, `42`)
checkJSON(t, `-0B_10_1010`, `-42`)
checkJSON(t, `+0B10_10_10`, `42`)

checkJSON(t, `0_600`, `384`)
checkJSON(t, `-0_600`, `-384`)
checkJSON(t, `+0_600`, `384`)

checkJSON(t, `0o600`, `384`)
checkJSON(t, `0o_600`, `384`)
checkJSON(t, `-0o600`, `-384`)
checkJSON(t, `-0o6_00`, `-384`)
checkJSON(t, `+0o600`, `384`)
checkJSON(t, `+0o60_0`, `384`)

checkJSON(t, `0O600`, `384`)
checkJSON(t, `0O_600`, `384`)
checkJSON(t, `-0O600`, `-384`)
checkJSON(t, `-0O6_00`, `-384`)
checkJSON(t, `+0O600`, `384`)
checkJSON(t, `+0O60_0`, `384`)

checkJSON(t, `0xBad_Face`, `195951310`)
checkJSON(t, `-0x_Bad_Face`, `-195951310`)
checkJSON(t, `+0xBad_Face`, `195951310`)

checkJSON(t, `0XBad_Face`, `195951310`)
checkJSON(t, `-0X_Bad_Face`, `-195951310`)
checkJSON(t, `+0XBad_Face`, `195951310`)

// as float64
checkJSON(t, `0_600.123`, `600.123`) // float64 can not be an octal number
checkJSON(t, `1_5.`, `15`)
checkJSON(t, `0.15e+0_2`, `15`)
checkJSON(t, `0x1p-2`, `0.25`)
checkJSON(t, `0x2.p10`, `2048`)
checkJSON(t, `0x1.Fp+0`, `1.9375`)
checkJSON(t, `0X.8p-0`, `0.5`)
checkJSON(t, `0X_1FFFP-16`, `0.1249847412109375`)
}
70 changes: 45 additions & 25 deletions internal/json/parser_test.go
View file
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
// Copyright (c) 2020, 2021, Maxime Soulé
// Copyright (c) 2020-2022, Maxime Soulé
// All rights reserved.
//
// This source code is licensed under the BSD-style license found in the
Expand All @@ -19,6 +19,28 @@ import (
"github.com/maxatome/go-testdeep/internal/test"
)

func checkJSON(t *testing.T, gotJSON, expectedJSON string) {
t.Helper()

var expected interface{}
err := ejson.Unmarshal([]byte(expectedJSON), &expected)
if err != nil {
t.Fatalf("bad JSON: %s", err)
}

got, err := json.Parse([]byte(gotJSON))
if !test.NoError(t, err, "json.Parse succeeds") {
return
}
if !reflect.DeepEqual(got, expected) {
test.EqualErrorMessage(t,
strings.TrimRight(spew.Sdump(got), "\n"),
strings.TrimRight(spew.Sdump(expected), "\n"),
"got matches expected",
)
}
}

func TestJSON(t *testing.T) {
t.Run("Basics", func(t *testing.T) {
for i, js := range []string{
Expand Down Expand Up @@ -83,31 +105,29 @@ func TestJSON(t *testing.T) {
})

t.Run("JSON spec infringements", func(t *testing.T) {
check := func(gotJSON, expectedJSON string) {
t.Helper()
var expected interface{}
err := ejson.Unmarshal([]byte(expectedJSON), &expected)
if err != nil {
t.Fatalf("bad JSON: %s", err)
}

got, err := json.Parse([]byte(gotJSON))
if !test.NoError(t, err, "json.Parse succeeds") {
return
}
if !reflect.DeepEqual(got, expected) {
test.EqualErrorMessage(t,
strings.TrimRight(spew.Sdump(got), "\n"),
strings.TrimRight(spew.Sdump(expected), "\n"),
"got matches expected",
)
}
}
// "," is accepted just before non-empty "}" or "]"
check(`{"foo": "bar", }`, `{"foo":"bar"}`)
check(`{"foo":"bar",}`, `{"foo":"bar"}`)
check(`[ 1, 2, 3, ]`, `[1,2,3]`)
check(`[ 1,2,3,]`, `[1,2,3]`)
checkJSON(t, `{"foo": "bar", }`, `{"foo":"bar"}`)
checkJSON(t, `{"foo":"bar",}`, `{"foo":"bar"}`)
checkJSON(t, `[ 1, 2, 3, ]`, `[1,2,3]`)
checkJSON(t, `[ 1,2,3,]`, `[1,2,3]`)

// Extend to golang accepted numbers
// as int64
checkJSON(t, `+42`, `42`)

checkJSON(t, `0600`, `384`)
checkJSON(t, `-0600`, `-384`)
checkJSON(t, `+0600`, `384`)

checkJSON(t, `0xBadFace`, `195951310`)
checkJSON(t, `-0xBadFace`, `-195951310`)
checkJSON(t, `+0xBadFace`, `195951310`)

// as float64
checkJSON(t, `0600.123`, `600.123`) // float64 can not be an octal number
checkJSON(t, `0600.`, `600`) // float64 can not be an octal number
checkJSON(t, `.25`, `0.25`)
checkJSON(t, `+123.`, `123`)
})

t.Run("Special string cases", func(t *testing.T) {
Expand Down
15 changes: 15 additions & 0 deletions td/td_json.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -581,6 +581,11 @@ func jsonify(ctx ctxerr.Context, got reflect.Value) (interface{}, *ctxerr.Error)
// - multi-lines comments start with the character sequence /* and stop
// with the first subsequent character sequence */.
//
// Other JSON divergences:
// - ',' can precede a '}' or a ']' (as in go);
// - int_lit & float_lit numbers as defined in go spec are accepted;
// - numbers can be prefixed by '+'.
//
// Most operators can be directly embedded in JSON without requiring
// any placeholder.
//
Expand Down Expand Up @@ -841,6 +846,11 @@ var _ TestDeep = &tdMapJSON{}
// - multi-lines comments start with the character sequence /* and stop
// with the first subsequent character sequence */.
//
// Other JSON divergences:
// - ',' can precede a '}' or a ']' (as in go);
// - int_lit & float_lit numbers as defined in go spec are accepted;
// - numbers can be prefixed by '+'.
//
// Most operators can be directly embedded in SubJSONOf without requiring
// any placeholder.
//
Expand Down Expand Up @@ -1055,6 +1065,11 @@ func SubJSONOf(expectedJSON interface{}, params ...interface{}) TestDeep {
// - multi-lines comments start with the character sequence /* and stop
// with the first subsequent character sequence */.
//
// Other JSON divergences:
// - ',' can precede a '}' or a ']' (as in go);
// - int_lit & float_lit numbers as defined in go spec are accepted;
// - numbers can be prefixed by '+'.
//
// Most operators can be directly embedded in SuperJSONOf without requiring
// any placeholder.
//
Expand Down

0 comments on commit 4aa5af6

Please sign in to comment.