BlackForestBytes/goext
20
0
Fork 0
Code Actions

gojson: added MarshalSafeCollections

Browse Source
This commit is contained in:
Mike Schwörer 2023-04-20 14:34:57 +02:00
parent d780c7965f
commit ef3705937c
Signed by: Mikescher
GPG Key ID: D3C7172E0A70F8CF
7 changed files with 167 additions and 549 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

27
gojson/LICENSE Normal file
View File

@ -0,0 +1,27 @@
Copyright (c) 2009 The Go Authors. All rights reserved.
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are
met:
* Redistributions of source code must retain the above copyright
notice, this list of conditions and the following disclaimer.
* Redistributions in binary form must reproduce the above
copyright notice, this list of conditions and the following disclaimer
in the documentation and/or other materials provided with the
distribution.
* Neither the name of Google Inc. nor the names of its
contributors may be used to endorse or promote products derived from
this software without specific prior written permission.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

8
gojson/README.md Normal file
View File

@ -0,0 +1,8 @@
JSON serializer which serializes nil-Arrays as `[]` and nil-maps als `{}`.
Idea from: https://github.com/homelight/json
Forked from https://github.com/golang/go/tree/547e8e22fe565d65d1fd4d6e71436a5a855447b0/src/encoding/json ( tag go1.20.2 )

541
gojson/bench_test.go
View File

@ -1,541 +0,0 @@
// Copyright 2011 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
// Large data benchmark.
// The JSON data is a summary of agl's changes in the
// go, webkit, and chromium open source projects.
// We benchmark converting between the JSON form
// and in-memory data structures.
package json
import (
"bytes"
"compress/gzip"
"fmt"
"internal/testenv"
"io"
"os"
"reflect"
"regexp"
"runtime"
"strings"
"sync"
"testing"
)
type codeResponse struct {
Tree *codeNode `json:"tree"`
Username string `json:"username"`
}
type codeNode struct {
Name string `json:"name"`
Kids []*codeNode `json:"kids"`
CLWeight float64 `json:"cl_weight"`
Touches int `json:"touches"`
MinT int64 `json:"min_t"`
MaxT int64 `json:"max_t"`
MeanT int64 `json:"mean_t"`
}
var codeJSON []byte
var codeStruct codeResponse
func codeInit() {
f, err := os.Open("testdata/code.json.gz")
if err != nil {
panic(err)
}
defer f.Close()
gz, err := gzip.NewReader(f)
if err != nil {
panic(err)
}
data, err := io.ReadAll(gz)
if err != nil {
panic(err)
}
codeJSON = data
if err := Unmarshal(codeJSON, &codeStruct); err != nil {
panic("unmarshal code.json: " + err.Error())
}
if data, err = Marshal(&codeStruct); err != nil {
panic("marshal code.json: " + err.Error())
}
if !bytes.Equal(data, codeJSON) {
println("different lengths", len(data), len(codeJSON))
for i := 0; i < len(data) && i < len(codeJSON); i++ {
if data[i] != codeJSON[i] {
println("re-marshal: changed at byte", i)
println("orig: ", string(codeJSON[i-10:i+10]))
println("new: ", string(data[i-10:i+10]))
break
}
}
panic("re-marshal code.json: different result")
}
}
func BenchmarkCodeEncoder(b *testing.B) {
b.ReportAllocs()
if codeJSON == nil {
b.StopTimer()
codeInit()
b.StartTimer()
}
b.RunParallel(func(pb *testing.PB) {
enc := NewEncoder(io.Discard)
for pb.Next() {
if err := enc.Encode(&codeStruct); err != nil {
b.Fatal("Encode:", err)
}
}
})
b.SetBytes(int64(len(codeJSON)))
}
func BenchmarkCodeEncoderError(b *testing.B) {
b.ReportAllocs()
if codeJSON == nil {
b.StopTimer()
codeInit()
b.StartTimer()
}
// Trigger an error in Marshal with cyclic data.
type Dummy struct {
Name string
Next *Dummy
}
dummy := Dummy{Name: "Dummy"}
dummy.Next = &dummy
b.RunParallel(func(pb *testing.PB) {
enc := NewEncoder(io.Discard)
for pb.Next() {
if err := enc.Encode(&codeStruct); err != nil {
b.Fatal("Encode:", err)
}
if _, err := Marshal(dummy); err == nil {
b.Fatal("expect an error here")
}
}
})
b.SetBytes(int64(len(codeJSON)))
}
func BenchmarkCodeMarshal(b *testing.B) {
b.ReportAllocs()
if codeJSON == nil {
b.StopTimer()
codeInit()
b.StartTimer()
}
b.RunParallel(func(pb *testing.PB) {
for pb.Next() {
if _, err := Marshal(&codeStruct); err != nil {
b.Fatal("Marshal:", err)
}
}
})
b.SetBytes(int64(len(codeJSON)))
}
func BenchmarkCodeMarshalError(b *testing.B) {
b.ReportAllocs()
if codeJSON == nil {
b.StopTimer()
codeInit()
b.StartTimer()
}
// Trigger an error in Marshal with cyclic data.
type Dummy struct {
Name string
Next *Dummy
}
dummy := Dummy{Name: "Dummy"}
dummy.Next = &dummy
b.RunParallel(func(pb *testing.PB) {
for pb.Next() {
if _, err := Marshal(&codeStruct); err != nil {
b.Fatal("Marshal:", err)
}
if _, err := Marshal(dummy); err == nil {
b.Fatal("expect an error here")
}
}
})
b.SetBytes(int64(len(codeJSON)))
}
func benchMarshalBytes(n int) func(*testing.B) {
sample := []byte("hello world")
// Use a struct pointer, to avoid an allocation when passing it as an
// interface parameter to Marshal.
v := &struct {
Bytes []byte
}{
bytes.Repeat(sample, (n/len(sample))+1)[:n],
}
return func(b *testing.B) {
for i := 0; i < b.N; i++ {
if _, err := Marshal(v); err != nil {
b.Fatal("Marshal:", err)
}
}
}
}
func benchMarshalBytesError(n int) func(*testing.B) {
sample := []byte("hello world")
// Use a struct pointer, to avoid an allocation when passing it as an
// interface parameter to Marshal.
v := &struct {
Bytes []byte
}{
bytes.Repeat(sample, (n/len(sample))+1)[:n],
}
// Trigger an error in Marshal with cyclic data.
type Dummy struct {
Name string
Next *Dummy
}
dummy := Dummy{Name: "Dummy"}
dummy.Next = &dummy
return func(b *testing.B) {
for i := 0; i < b.N; i++ {
if _, err := Marshal(v); err != nil {
b.Fatal("Marshal:", err)
}
if _, err := Marshal(dummy); err == nil {
b.Fatal("expect an error here")
}
}
}
}
func BenchmarkMarshalBytes(b *testing.B) {
b.ReportAllocs()
// 32 fits within encodeState.scratch.
b.Run("32", benchMarshalBytes(32))
// 256 doesn't fit in encodeState.scratch, but is small enough to
// allocate and avoid the slower base64.NewEncoder.
b.Run("256", benchMarshalBytes(256))
// 4096 is large enough that we want to avoid allocating for it.
b.Run("4096", benchMarshalBytes(4096))
}
func BenchmarkMarshalBytesError(b *testing.B) {
b.ReportAllocs()
// 32 fits within encodeState.scratch.
b.Run("32", benchMarshalBytesError(32))
// 256 doesn't fit in encodeState.scratch, but is small enough to
// allocate and avoid the slower base64.NewEncoder.
b.Run("256", benchMarshalBytesError(256))
// 4096 is large enough that we want to avoid allocating for it.
b.Run("4096", benchMarshalBytesError(4096))
}
func BenchmarkCodeDecoder(b *testing.B) {
b.ReportAllocs()
if codeJSON == nil {
b.StopTimer()
codeInit()
b.StartTimer()
}
b.RunParallel(func(pb *testing.PB) {
var buf bytes.Buffer
dec := NewDecoder(&buf)
var r codeResponse
for pb.Next() {
buf.Write(codeJSON)
// hide EOF
buf.WriteByte('\n')
buf.WriteByte('\n')
buf.WriteByte('\n')
if err := dec.Decode(&r); err != nil {
b.Fatal("Decode:", err)
}
}
})
b.SetBytes(int64(len(codeJSON)))
}
func BenchmarkUnicodeDecoder(b *testing.B) {
b.ReportAllocs()
j := []byte(`"\uD83D\uDE01"`)
b.SetBytes(int64(len(j)))
r := bytes.NewReader(j)
dec := NewDecoder(r)
var out string
b.ResetTimer()
for i := 0; i < b.N; i++ {
if err := dec.Decode(&out); err != nil {
b.Fatal("Decode:", err)
}
r.Seek(0, 0)
}
}
func BenchmarkDecoderStream(b *testing.B) {
b.ReportAllocs()
b.StopTimer()
var buf bytes.Buffer
dec := NewDecoder(&buf)
buf.WriteString(`"` + strings.Repeat("x", 1000000) + `"` + "\n\n\n")
var x any
if err := dec.Decode(&x); err != nil {
b.Fatal("Decode:", err)
}
ones := strings.Repeat(" 1\n", 300000) + "\n\n\n"
b.StartTimer()
for i := 0; i < b.N; i++ {
if i%300000 == 0 {
buf.WriteString(ones)
}
x = nil
if err := dec.Decode(&x); err != nil || x != 1.0 {
b.Fatalf("Decode: %v after %d", err, i)
}
}
}
func BenchmarkCodeUnmarshal(b *testing.B) {
b.ReportAllocs()
if codeJSON == nil {
b.StopTimer()
codeInit()
b.StartTimer()
}
b.RunParallel(func(pb *testing.PB) {
for pb.Next() {
var r codeResponse
if err := Unmarshal(codeJSON, &r); err != nil {
b.Fatal("Unmarshal:", err)
}
}
})
b.SetBytes(int64(len(codeJSON)))
}
func BenchmarkCodeUnmarshalReuse(b *testing.B) {
b.ReportAllocs()
if codeJSON == nil {
b.StopTimer()
codeInit()
b.StartTimer()
}
b.RunParallel(func(pb *testing.PB) {
var r codeResponse
for pb.Next() {
if err := Unmarshal(codeJSON, &r); err != nil {
b.Fatal("Unmarshal:", err)
}
}
})
b.SetBytes(int64(len(codeJSON)))
}
func BenchmarkUnmarshalString(b *testing.B) {
b.ReportAllocs()
data := []byte(`"hello, world"`)
b.RunParallel(func(pb *testing.PB) {
var s string
for pb.Next() {
if err := Unmarshal(data, &s); err != nil {
b.Fatal("Unmarshal:", err)
}
}
})
}
func BenchmarkUnmarshalFloat64(b *testing.B) {
b.ReportAllocs()
data := []byte(`3.14`)
b.RunParallel(func(pb *testing.PB) {
var f float64
for pb.Next() {
if err := Unmarshal(data, &f); err != nil {
b.Fatal("Unmarshal:", err)
}
}
})
}
func BenchmarkUnmarshalInt64(b *testing.B) {
b.ReportAllocs()
data := []byte(`3`)
b.RunParallel(func(pb *testing.PB) {
var x int64
for pb.Next() {
if err := Unmarshal(data, &x); err != nil {
b.Fatal("Unmarshal:", err)
}
}
})
}
func BenchmarkIssue10335(b *testing.B) {
b.ReportAllocs()
j := []byte(`{"a":{ }}`)
b.RunParallel(func(pb *testing.PB) {
var s struct{}
for pb.Next() {
if err := Unmarshal(j, &s); err != nil {
b.Fatal(err)
}
}
})
}
func BenchmarkIssue34127(b *testing.B) {
b.ReportAllocs()
j := struct {
Bar string `json:"bar,string"`
}{
Bar: `foobar`,
}
b.RunParallel(func(pb *testing.PB) {
for pb.Next() {
if _, err := Marshal(&j); err != nil {
b.Fatal(err)
}
}
})
}
func BenchmarkUnmapped(b *testing.B) {
b.ReportAllocs()
j := []byte(`{"s": "hello", "y": 2, "o": {"x": 0}, "a": [1, 99, {"x": 1}]}`)
b.RunParallel(func(pb *testing.PB) {
var s struct{}
for pb.Next() {
if err := Unmarshal(j, &s); err != nil {
b.Fatal(err)
}
}
})
}
func BenchmarkTypeFieldsCache(b *testing.B) {
b.ReportAllocs()
var maxTypes int = 1e6
if testenv.Builder() != "" {
maxTypes = 1e3 // restrict cache sizes on builders
}
// Dynamically generate many new types.
types := make([]reflect.Type, maxTypes)
fs := []reflect.StructField{{
Type: reflect.TypeOf(""),
Index: []int{0},
}}
for i := range types {
fs[0].Name = fmt.Sprintf("TypeFieldsCache%d", i)
types[i] = reflect.StructOf(fs)
}
// clearClear clears the cache. Other JSON operations, must not be running.
clearCache := func() {
fieldCache = sync.Map{}
}
// MissTypes tests the performance of repeated cache misses.
// This measures the time to rebuild a cache of size nt.
for nt := 1; nt <= maxTypes; nt *= 10 {
ts := types[:nt]
b.Run(fmt.Sprintf("MissTypes%d", nt), func(b *testing.B) {
nc := runtime.GOMAXPROCS(0)
for i := 0; i < b.N; i++ {
clearCache()
var wg sync.WaitGroup
for j := 0; j < nc; j++ {
wg.Add(1)
go func(j int) {
for _, t := range ts[(j*len(ts))/nc : ((j+1)*len(ts))/nc] {
cachedTypeFields(t)
}
wg.Done()
}(j)
}
wg.Wait()
}
})
}
// HitTypes tests the performance of repeated cache hits.
// This measures the average time of each cache lookup.
for nt := 1; nt <= maxTypes; nt *= 10 {
// Pre-warm a cache of size nt.
clearCache()
for _, t := range types[:nt] {
cachedTypeFields(t)
}
b.Run(fmt.Sprintf("HitTypes%d", nt), func(b *testing.B) {
b.RunParallel(func(pb *testing.PB) {
for pb.Next() {
cachedTypeFields(types[0])
}
})
})
}
}
func BenchmarkEncodeMarshaler(b *testing.B) {
b.ReportAllocs()
m := struct {
A int
B RawMessage
}{}
b.RunParallel(func(pb *testing.PB) {
enc := NewEncoder(io.Discard)
for pb.Next() {
if err := enc.Encode(&m); err != nil {
b.Fatal("Encode:", err)
}
}
})
}
func BenchmarkEncoderEncode(b *testing.B) {
b.ReportAllocs()
type T struct {
X, Y string
}
v := &T{"foo", "bar"}
b.RunParallel(func(pb *testing.PB) {
for pb.Next() {
if err := NewEncoder(io.Discard).Encode(v); err != nil {
b.Fatal(err)
}
}
})
}
func BenchmarkNumberIsValid(b *testing.B) {
s := "-61657.61667E+61673"
for i := 0; i < b.N; i++ {
isValidNumber(s)
}
}
func BenchmarkNumberIsValidRegexp(b *testing.B) {
var jsonNumberRegexp = regexp.MustCompile(`^-?(?:0|[1-9]\d*)(?:\.\d+)?(?:[eE][+-]?\d+)?$`)
s := "-61657.61667E+61673"
for i := 0; i < b.N; i++ {
jsonNumberRegexp.MatchString(s)
}
}

29
gojson/encode.go
View File

@ -167,6 +167,17 @@ func Marshal(v any) ([]byte, error) {
return buf, nil
}
// MarshalSafeCollections is like Marshal except it will marshal nil maps and
// slices as '{}' and '[]' respectfully instead of 'null'
func MarshalSafeCollections(v interface{}, nilSafeSlices bool, nilSafeMaps bool) ([]byte, error) {
e := &encodeState{}
err := e.marshal(v, encOpts{escapeHTML: true, nilSafeSlices: nilSafeSlices, nilSafeMaps: nilSafeMaps})
if err != nil {
return nil, err
}
return e.Bytes(), nil
}
// MarshalIndent is like Marshal but applies Indent to format the output.
// Each JSON element in the output will begin on a new line beginning with prefix
// followed by one or more copies of indent according to the indentation nesting.
@ -363,6 +374,10 @@ type encOpts struct {
quoted bool
// escapeHTML causes '<', '>', and '&' to be escaped in JSON strings.
escapeHTML bool
// nilSafeSlices marshals a nil slices into '[]' instead of 'null'
nilSafeSlices bool
// nilSafeMaps marshals a nil maps '{}' instead of 'null'
nilSafeMaps bool
}
type encoderFunc func(e *encodeState, v reflect.Value, opts encOpts)
@ -776,7 +791,11 @@ type mapEncoder struct {
func (me mapEncoder) encode(e *encodeState, v reflect.Value, opts encOpts) {
if v.IsNil() {
if opts.nilSafeMaps {
e.WriteString("{}")
} else {
e.WriteString("null")
}
return
}
if e.ptrLevel++; e.ptrLevel > startDetectingCyclesAfter {
@ -829,9 +848,13 @@ func newMapEncoder(t reflect.Type) encoderFunc {
return me.encode
}
func encodeByteSlice(e *encodeState, v reflect.Value, _ encOpts) {
func encodeByteSlice(e *encodeState, v reflect.Value, opts encOpts) {
if v.IsNil() {
if opts.nilSafeSlices {
e.WriteString(`""`)
} else {
e.WriteString("null")
}
return
}
s := v.Bytes()
@ -866,7 +889,11 @@ type sliceEncoder struct {
func (se sliceEncoder) encode(e *encodeState, v reflect.Value, opts encOpts) {
if v.IsNil() {
if opts.nilSafeSlices {
e.WriteString("[]")
} else {
e.WriteString("null")
}
return
}
if e.ptrLevel++; e.ptrLevel > startDetectingCyclesAfter {

46
gojson/encode_test.go
View File

@ -1237,3 +1237,49 @@ func TestMarshalerError(t *testing.T) {
}
}
}
func TestMarshalSafeCollections(t *testing.T) {
var (
nilSlice []interface{}
pNilSlice *[]interface{}
nilMap map[string]interface{}
pNilMap *map[string]interface{}
)
type (
nilSliceStruct struct {
NilSlice []interface{} `json:"nil_slice"`
}
nilMapStruct struct {
NilMap map[string]interface{} `json:"nil_map"`
}
)
tests := []struct {
in interface{}
want string
}{
{nilSlice, "[]"},
{[]interface{}{}, "[]"},
{make([]interface{}, 0), "[]"},
{[]int{1, 2, 3}, "[1,2,3]"},
{pNilSlice, "null"},
{nilSliceStruct{}, "{\"nil_slice\":[]}"},
{nilMap, "{}"},
{map[string]interface{}{}, "{}"},
{make(map[string]interface{}, 0), "{}"},
{map[string]interface{}{"1": 1, "2": 2, "3": 3}, "{\"1\":1,\"2\":2,\"3\":3}"},
{pNilMap, "null"},
{nilMapStruct{}, "{\"nil_map\":{}}"},
}
for i, tt := range tests {
b, err := MarshalSafeCollections(tt.in, true, true)
if err != nil {
t.Errorf("test %d, unexpected failure: %v", i, err)
}
if got := string(b); got != tt.want {
t.Errorf("test %d, Marshal(%#v) = %q, want %q", i, tt.in, got, tt.want)
}
}
}

11
gojson/stream.go
View File

@ -182,6 +182,8 @@ type Encoder struct {
w io.Writer
err error
escapeHTML bool
nilSafeSlices bool
nilSafeMaps bool
indentBuf *bytes.Buffer
indentPrefix string
@ -206,7 +208,7 @@ func (enc *Encoder) Encode(v any) error {
e := newEncodeState()
defer encodeStatePool.Put(e)
err := e.marshal(v, encOpts{escapeHTML: enc.escapeHTML})
err := e.marshal(v, encOpts{escapeHTML: enc.escapeHTML, nilSafeMaps: enc.nilSafeMaps, nilSafeSlices: enc.nilSafeSlices})
if err != nil {
return err
}
@ -245,6 +247,13 @@ func (enc *Encoder) SetIndent(prefix, indent string) {
enc.indentValue = indent
}
// SetNilSafeCollection specifies whether to represent nil slices and maps as
// '[]' or '{}' respectfully (flag on) instead of 'null' (default) when marshaling json.
func (enc *Encoder) SetNilSafeCollection(nilSafeSlices bool, nilSafeMaps bool) {
enc.nilSafeSlices = nilSafeSlices
enc.nilSafeMaps = nilSafeMaps
}
// SetEscapeHTML specifies whether problematic HTML characters
// should be escaped inside JSON quoted strings.
// The default behavior is to escape &, <, and > to \u0026, \u003c, and \u003e

42
gojson/stream_test.go
View File

@ -495,3 +495,45 @@ func TestHTTPDecoding(t *testing.T) {
t.Errorf("err = %v; want io.EOF", err)
}
}
func TestEncoderSetNilSafeCollection(t *testing.T) {
var (
nilSlice []interface{}
pNilSlice *[]interface{}
nilMap map[string]interface{}
pNilMap *map[string]interface{}
)
for _, tt := range []struct {
name string
v interface{}
want string
rescuedWant string
}{
{"nilSlice", nilSlice, "null", "[]"},
{"nonNilSlice", []interface{}{}, "[]", "[]"},
{"sliceWithValues", []interface{}{1, 2, 3}, "[1,2,3]", "[1,2,3]"},
{"pNilSlice", pNilSlice, "null", "null"},
{"nilMap", nilMap, "null", "{}"},
{"nonNilMap", map[string]interface{}{}, "{}", "{}"},
{"mapWithValues", map[string]interface{}{"1": 1, "2": 2, "3": 3}, "{\"1\":1,\"2\":2,\"3\":3}", "{\"1\":1,\"2\":2,\"3\":3}"},
{"pNilMap", pNilMap, "null", "null"},
} {
var buf bytes.Buffer
enc := NewEncoder(&buf)
if err := enc.Encode(tt.v); err != nil {
t.Fatalf("Encode(%s): %s", tt.name, err)
}
if got := strings.TrimSpace(buf.String()); got != tt.want {
t.Errorf("Encode(%s) = %#q, want %#q", tt.name, got, tt.want)
}
buf.Reset()
enc.SetNilSafeCollection(true, true)
if err := enc.Encode(tt.v); err != nil {
t.Fatalf("SetNilSafeCollection(true) Encode(%s): %s", tt.name, err)
}
if got := strings.TrimSpace(buf.String()); got != tt.rescuedWant {
t.Errorf("SetNilSafeCollection(true) Encode(%s) = %#q, want %#q",
tt.name, got, tt.want)
}
}
}