8000 perf(libs/json): Lower heap overhead of JSON encoding (backport #2846) by mergify[bot] · Pull Request #2874 · cometbft/cometbft · GitHub
[go: up one dir, main page]
More Web Proxy on the site http://driver.im/
Skip to content

perf(libs/json): Lower heap overhead of JSON encoding (backport #2846) #2874

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 1 commit into from
Apr 23, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
@@ -0,0 +1,2 @@
- `[libs/json]` Lower the memory overhead of JSON encoding by using JSON encoders internally
([\#2846](https://github.com/cometbft/cometbft/pull/2846)).
25 changes: 14 additions & 11 deletions libs/json/encoder.go
Original file line number Diff line number Diff line change
Expand Up @@ -42,7 +42,7 @@ func MarshalIndent(v any, prefix, indent string) ([]byte, error) {
return buf.Bytes(), nil
}

func encode(w io.Writer, v any) error {
func encode(w *bytes.Buffer, v any) error {
// Bare nil values can't be reflected, so we must handle them here.
if v == nil {
return writeStr(w, "null")
Expand All @@ -60,7 +60,7 @@ func encode(w io.Writer, v any) error {
return encodeReflect(w, rv)
}

func encodeReflect(w io.Writer, rv reflect.Value) error {
func encodeReflect(w *bytes.Buffer, rv reflect.Value) error {
if !rv.IsValid() {
return errors.New("invalid reflect value")
}
Expand Down Expand Up @@ -115,7 +115,7 @@ func encodeReflect(w io.Writer, rv reflect.Value) error {
}
}

func encodeReflectList(w io.Writer, rv reflect.Value) error {
func encodeReflectList(w *bytes.Buffer, rv reflect.Value) error {
// Emit nil slices as null.
if rv.Kind() == reflect.Slice && rv.IsNil() {
return writeStr(w, "null")
Expand Down Expand Up @@ -150,7 +150,7 @@ func encodeReflectList(w io.Writer, rv reflect.Value) error {
return writeStr(w, "]")
}

func encodeReflectMap(w io.Writer, rv reflect.Value) error {
func encodeReflectMap(w *bytes.Buffer, rv reflect.Value) error {
if rv.Type().Key().Kind() != reflect.String {
return errors.New("map key must be string")
}
Expand Down Expand Up @@ -181,7 +181,7 @@ func encodeReflectMap(w io.Writer, rv reflect.Value) error {
return writeStr(w, "}")
}

func encodeReflectStruct(w io.Writer, rv reflect.Value) error {
func encodeReflectStruct(w *bytes.Buffer, rv reflect.Value) error {
sInfo := makeStructInfo(rv.Type())
if err := writeStr(w, "{"); err != nil {
return err
Expand Down Expand Up @@ -212,7 +212,7 @@ func encodeReflectStruct(w io.Writer, rv reflect.Value) error {
return writeStr(w, "}")
}

func encodeReflectInterface(w io.Writer, rv reflect.Value) error {
func encodeReflectInterface(w *bytes.Buffer, rv reflect.Value) error {
// Get concrete value and dereference pointers.
for rv.Kind() == reflect.Ptr || rv.Kind() == reflect.Interface {
if rv.IsNil() {
Expand All @@ -237,14 +237,17 @@ func encodeReflectInterface(w io.Writer, rv reflect.Value) error {
return writeStr(w, "}")
}

func encodeStdlib(w io.Writer, v any) error {
// Doesn't stream the output because that adds a newline, as per:
// https://golang.org/pkg/encoding/json/#Encoder.Encode
blob, err := json.Marshal(v)
func encodeStdlib(w *bytes.Buffer, v any) error {
// Stream the output of the JSON marshaling directly into the buffer.
// The stdlib encoder will write a newline, so we must truncate it,
// which is why we pass in a bytes.Buffer throughout, not io.Writer.
enc := json.NewEncoder(w)
err := enc.Encode(v)
if err != nil {
return err
}
_, err = w.Write(blob)
// Remove the last byte from the buffer
w.Truncate(w.Len() - 1)
return err
}

Expand Down
17 changes: 17 additions & 0 deletions libs/json/encoder_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -102,3 +102,20 @@ func TestMarshal(t *testing.T) {
})
}
}

func BenchmarkJsonMarshalStruct(b *testing.B) {
s := "string"
sPtr := &s
i64 := int64(64)
ti := time.Date(2020, 6, 2, 18, 5, 13, 4346374, time.FixedZone("UTC+2", 2*60*60))
car := &Car{Wheels: 4}
boat := Boat{Sail: true}
for i := 0; i < b.N; i++ {
_, _ = json.Marshal(Struct{
Bool: true, Float64: 3.14, Int32: 32, Int64: 64, Int64Ptr: &i64,
String: "foo", StringPtrPtr: &sPtr, Bytes: []byte{1, 2, 3},
Time: ti, Car: car, Boat: boat, Vehicles: []Vehicle{car, boat},
Child: &Struct{Bool: false, String: "child"}, private: "private",
})
}
}
Loading
0