From 0f52b860ea9c44e050473bc809f2cb72de73b62b Mon Sep 17 00:00:00 2001 From: Timo Vetter Date: Thu, 5 Oct 2023 10:57:34 +0200 Subject: [PATCH] DYN-166 add jsonfilter to json library --- gojson/encode.go | 3 +- gojson/fold.go | 5 ++- gojson/fold_test.go | 12 +++-- gojson/number_test.go | 15 +++++++ gojson/scanner.go | 2 +- gojson/stream.go | 21 +++------ gojson/stream_test.go | 101 ++++++++---------------------------------- 7 files changed, 54 insertions(+), 105 deletions(-) diff --git a/gojson/encode.go b/gojson/encode.go index 765e30e..346cbc1 100644 --- a/gojson/encode.go +++ b/gojson/encode.go @@ -156,7 +156,6 @@ import ( // an error. func Marshal(v any) ([]byte, error) { e := newEncodeState() - defer encodeStatePool.Put(e) err := e.marshal(v, encOpts{escapeHTML: true}) if err != nil { @@ -164,6 +163,8 @@ func Marshal(v any) ([]byte, error) { } buf := append([]byte(nil), e.Bytes()...) + encodeStatePool.Put(e) + return buf, nil } diff --git a/gojson/fold.go b/gojson/fold.go index 0f9b09d..ab249b2 100644 --- a/gojson/fold.go +++ b/gojson/fold.go @@ -97,7 +97,10 @@ func equalFoldRight(s, t []byte) bool { t = t[size:] } - return len(t) == 0 + if len(t) > 0 { + return false + } + return true } // asciiEqualFold is a specialization of bytes.EqualFold for use when diff --git a/gojson/fold_test.go b/gojson/fold_test.go index 4daa359..9fb9464 100644 --- a/gojson/fold_test.go +++ b/gojson/fold_test.go @@ -52,7 +52,9 @@ func TestFold(t *testing.T) { } func TestFoldAgainstUnicode(t *testing.T) { - var buf1, buf2 []byte + const bufSize = 5 + buf1 := make([]byte, 0, bufSize) + buf2 := make([]byte, 0, bufSize) var runes []rune for i := 0x20; i <= 0x7f; i++ { runes = append(runes, rune(i)) @@ -94,8 +96,12 @@ func TestFoldAgainstUnicode(t *testing.T) { continue } for _, r2 := range runes { - buf1 = append(utf8.AppendRune(append(buf1[:0], 'x'), r), 'x') - buf2 = append(utf8.AppendRune(append(buf2[:0], 'x'), r2), 'x') + buf1 := append(buf1[:0], 'x') + buf2 := append(buf2[:0], 'x') + buf1 = buf1[:1+utf8.EncodeRune(buf1[1:bufSize], r)] + buf2 = buf2[:1+utf8.EncodeRune(buf2[1:bufSize], r2)] + buf1 = append(buf1, 'x') + buf2 = append(buf2, 'x') want := bytes.EqualFold(buf1, buf2) if got := ff.fold(buf1, buf2); got != want { t.Errorf("%s(%q, %q) = %v; want %v", ff.name, buf1, buf2, got, want) diff --git a/gojson/number_test.go b/gojson/number_test.go index c82e6de..cc67018 100644 --- a/gojson/number_test.go +++ b/gojson/number_test.go @@ -116,3 +116,18 @@ func TestNumberIsValid(t *testing.T) { } } } + +func BenchmarkNumberIsValid(b *testing.B) { + s := "-61657.61667E+61673" + for i := 0; i < b.N; i++ { + isValidNumber(s) + } +} + +func BenchmarkNumberIsValidRegexp(b *testing.B) { + var jsonNumberRegexp = regexp.MustCompile(`^-?(?:0|[1-9]\d*)(?:\.\d+)?(?:[eE][+-]?\d+)?$`) + s := "-61657.61667E+61673" + for i := 0; i < b.N; i++ { + jsonNumberRegexp.MatchString(s) + } +} diff --git a/gojson/scanner.go b/gojson/scanner.go index 4c43f5f..22fc692 100644 --- a/gojson/scanner.go +++ b/gojson/scanner.go @@ -594,7 +594,7 @@ func (s *scanner) error(c byte, context string) int { return scanError } -// quoteChar formats c as a quoted character literal. +// quoteChar formats c as a quoted character literal func quoteChar(c byte) string { // special cases - different from quoted strings if c == '\'' { diff --git a/gojson/stream.go b/gojson/stream.go index c5f368f..b278ee4 100644 --- a/gojson/stream.go +++ b/gojson/stream.go @@ -179,11 +179,9 @@ func nonSpace(b []byte) bool { // An Encoder writes JSON values to an output stream. type Encoder struct { - w io.Writer - err error - escapeHTML bool - nilSafeSlices bool - nilSafeMaps bool + w io.Writer + err error + escapeHTML bool indentBuf *bytes.Buffer indentPrefix string @@ -204,11 +202,8 @@ func (enc *Encoder) Encode(v any) error { if enc.err != nil { return enc.err } - e := newEncodeState() - defer encodeStatePool.Put(e) - - err := e.marshal(v, encOpts{escapeHTML: enc.escapeHTML, nilSafeMaps: enc.nilSafeMaps, nilSafeSlices: enc.nilSafeSlices}) + err := e.marshal(v, encOpts{escapeHTML: enc.escapeHTML}) if err != nil { return err } @@ -236,6 +231,7 @@ func (enc *Encoder) Encode(v any) error { if _, err = enc.w.Write(b); err != nil { enc.err = err } + encodeStatePool.Put(e) return err } @@ -247,13 +243,6 @@ func (enc *Encoder) SetIndent(prefix, indent string) { enc.indentValue = indent } -// SetNilSafeCollection specifies whether to represent nil slices and maps as -// '[]' or '{}' respectfully (flag on) instead of 'null' (default) when marshaling json. -func (enc *Encoder) SetNilSafeCollection(nilSafeSlices bool, nilSafeMaps bool) { - enc.nilSafeSlices = nilSafeSlices - enc.nilSafeMaps = nilSafeMaps -} - // SetEscapeHTML specifies whether problematic HTML characters // should be escaped inside JSON quoted strings. // The default behavior is to escape &, <, and > to \u0026, \u003c, and \u003e diff --git a/gojson/stream_test.go b/gojson/stream_test.go index 678ba0f..0e156d9 100644 --- a/gojson/stream_test.go +++ b/gojson/stream_test.go @@ -12,7 +12,6 @@ import ( "net/http" "net/http/httptest" "reflect" - "runtime/debug" "strings" "testing" ) @@ -42,7 +41,7 @@ false func TestEncoder(t *testing.T) { for i := 0; i <= len(streamTest); i++ { - var buf strings.Builder + var buf bytes.Buffer enc := NewEncoder(&buf) // Check that enc.SetIndent("", "") turns off indentation. enc.SetIndent(">", ".") @@ -60,43 +59,6 @@ func TestEncoder(t *testing.T) { } } -func TestEncoderErrorAndReuseEncodeState(t *testing.T) { - // Disable the GC temporarily to prevent encodeState's in Pool being cleaned away during the test. - percent := debug.SetGCPercent(-1) - defer debug.SetGCPercent(percent) - - // Trigger an error in Marshal with cyclic data. - type Dummy struct { - Name string - Next *Dummy - } - dummy := Dummy{Name: "Dummy"} - dummy.Next = &dummy - - var buf bytes.Buffer - enc := NewEncoder(&buf) - if err := enc.Encode(dummy); err == nil { - t.Errorf("Encode(dummy) == nil; want error") - } - - type Data struct { - A string - I int - } - data := Data{A: "a", I: 1} - if err := enc.Encode(data); err != nil { - t.Errorf("Marshal(%v) = %v", data, err) - } - - var data2 Data - if err := Unmarshal(buf.Bytes(), &data2); err != nil { - t.Errorf("Unmarshal(%v) = %v", data2, err) - } - if data2 != data { - t.Errorf("expect: %v, but get: %v", data, data2) - } -} - var streamEncodedIndent = `0.1 "hello" null @@ -115,7 +77,7 @@ false ` func TestEncoderIndent(t *testing.T) { - var buf strings.Builder + var buf bytes.Buffer enc := NewEncoder(&buf) enc.SetIndent(">", ".") for _, v := range streamTest { @@ -185,7 +147,7 @@ func TestEncoderSetEscapeHTML(t *testing.T) { `{"bar":"\"foobar\""}`, }, } { - var buf strings.Builder + var buf bytes.Buffer enc := NewEncoder(&buf) if err := enc.Encode(tt.v); err != nil { t.Errorf("Encode(%s): %s", tt.name, err) @@ -347,6 +309,21 @@ func TestBlocking(t *testing.T) { } } +func BenchmarkEncoderEncode(b *testing.B) { + b.ReportAllocs() + type T struct { + X, Y string + } + v := &T{"foo", "bar"} + b.RunParallel(func(pb *testing.PB) { + for pb.Next() { + if err := NewEncoder(io.Discard).Encode(v); err != nil { + b.Fatal(err) + } + } + }) +} + type tokenStreamCase struct { json string expTokens []any @@ -495,45 +472,3 @@ func TestHTTPDecoding(t *testing.T) { t.Errorf("err = %v; want io.EOF", err) } } - -func TestEncoderSetNilSafeCollection(t *testing.T) { - var ( - nilSlice []interface{} - pNilSlice *[]interface{} - nilMap map[string]interface{} - pNilMap *map[string]interface{} - ) - for _, tt := range []struct { - name string - v interface{} - want string - rescuedWant string - }{ - {"nilSlice", nilSlice, "null", "[]"}, - {"nonNilSlice", []interface{}{}, "[]", "[]"}, - {"sliceWithValues", []interface{}{1, 2, 3}, "[1,2,3]", "[1,2,3]"}, - {"pNilSlice", pNilSlice, "null", "null"}, - {"nilMap", nilMap, "null", "{}"}, - {"nonNilMap", map[string]interface{}{}, "{}", "{}"}, - {"mapWithValues", map[string]interface{}{"1": 1, "2": 2, "3": 3}, "{\"1\":1,\"2\":2,\"3\":3}", "{\"1\":1,\"2\":2,\"3\":3}"}, - {"pNilMap", pNilMap, "null", "null"}, - } { - var buf bytes.Buffer - enc := NewEncoder(&buf) - if err := enc.Encode(tt.v); err != nil { - t.Fatalf("Encode(%s): %s", tt.name, err) - } - if got := strings.TrimSpace(buf.String()); got != tt.want { - t.Errorf("Encode(%s) = %#q, want %#q", tt.name, got, tt.want) - } - buf.Reset() - enc.SetNilSafeCollection(true, true) - if err := enc.Encode(tt.v); err != nil { - t.Fatalf("SetNilSafeCollection(true) Encode(%s): %s", tt.name, err) - } - if got := strings.TrimSpace(buf.String()); got != tt.rescuedWant { - t.Errorf("SetNilSafeCollection(true) Encode(%s) = %#q, want %#q", - tt.name, got, tt.want) - } - } -}