Comments (5)
So far we have tried to decycle the incoming data, but seems have big impact on memory.
The code is:
package trace_marshal
import (
"reflect"
"unsafe"
)
// NOTE: this file is a temporiray backup
// decyclic seems has signaficant memory consumption
// making it slow to decyclic when encounters large data
type decyclicer struct {
seen map[uintptr]struct{}
}
func Decyclic(v interface{}) interface{} {
if v == nil {
return nil
}
d := &decyclicer{
seen: map[uintptr]struct{}{},
}
d.clear(reflect.ValueOf(v), func(r reflect.Value) {
v = r.Interface()
})
return v
}
func makeAddrable(v reflect.Value, set func(r reflect.Value)) reflect.Value {
if v.CanAddr() {
return v
}
p := reflect.New(v.Type())
p.Elem().Set(v)
x := p.Elem()
set(x)
return x
}
func (c *decyclicer) clear(v reflect.Value, set func(r reflect.Value)) {
// fmt.Printf("clear: %v\n", v.Type())
switch v.Kind() {
case reflect.Ptr:
if v.IsNil() {
return
}
// only pointer can create cyclic
ptr := v.Pointer()
if ptr == 0 {
return
}
if _, ok := c.seen[ptr]; ok {
// fmt.Printf("found : 0x%x -> %v\n", ptr, v.Interface())
set(reflect.Zero(v.Type()))
return
}
c.seen[ptr] = struct{}{}
defer delete(c.seen, ptr)
v = makeAddrable(v, set)
c.clear(v.Elem(), func(r reflect.Value) {
v.Elem().Set(r)
})
case reflect.Interface:
if v.IsNil() {
return
}
v = makeAddrable(v, set)
c.clear(v.Elem(), func(r reflect.Value) {
// NOTE: interface{} is special
// we can directly can call v.Set
// instead of v.Elem().Set()
v.Set(r)
if v.Elem().Kind() == reflect.Ptr && v.Elem().IsNil() {
// fmt.Printf("found isNil\n")
// avoid {nil-value,non-nil type}
set(reflect.Zero(v.Type()))
}
})
case reflect.Array, reflect.Slice:
switch v.Type().Elem().Kind() {
case reflect.Int64, reflect.Int, reflect.Int32, reflect.Int16,
reflect.Uint64, reflect.Uint, reflect.Uint32, reflect.Uint16,
reflect.Float64, reflect.Float32,
reflect.String,
reflect.Bool:
return
case reflect.Int8, reflect.Uint8:
// []byte -> Uint8
// ignore some: 10K JSON
n := v.Len()
if v.Kind() == reflect.Slice && n > 10*1024 {
// reserve first 16 and last 16
// S[:16] ... + S[len(S)-16:]
const reserve = 16
const ellipse = 3
const totalLen = reserve*2 + ellipse
newSlice := reflect.MakeSlice(v.Type(), totalLen, totalLen)
for i := 0; i < reserve; i++ {
newSlice.Index(i).Set(v.Index(i))
}
for i := 0; i < ellipse; i++ {
if v.Kind() == reflect.Uint8 {
newSlice.Index(reserve + i).SetUint('.')
} else if v.Kind() == reflect.Int8 {
newSlice.Index(reserve + i).SetInt('.')
}
}
for i := 0; i < reserve; i++ {
newSlice.Index(reserve + ellipse + i).Set(v.Index(n - reserve + i))
}
set(newSlice)
}
return
}
v = makeAddrable(v, set)
for i := 0; i < v.Len(); i++ {
e := v.Index(i)
c.clear(e, func(r reflect.Value) {
e.Set(r)
})
}
case reflect.Map:
v = makeAddrable(v, set)
iter := v.MapRange()
// sets := [][2]reflect.Value{}
for iter.Next() {
vi := v.MapIndex(iter.Key())
c.clear(vi, func(r reflect.Value) {
v.SetMapIndex(iter.Key(), r)
})
}
case reflect.Struct:
// fmt.Printf("struct \n")
// make struct addrable
v = makeAddrable(v, set)
for i := 0; i < v.NumField(); i++ {
field := v.Field(i)
if field.CanSet() {
c.clear(field, func(r reflect.Value) {
field.Set(r)
})
} else {
e := reflect.NewAt(field.Type(), unsafe.Pointer(field.UnsafeAddr()))
c.clear(e.Elem(), func(r reflect.Value) {
e.Elem().Set(r)
})
// panic(fmt.Errorf("cannot set: %v", field))
}
}
case reflect.Chan, reflect.Func:
// ignore
default:
// int
}
}
from xgo.
Solution to the cyclic stack trace problem: separate stack trace hierarchical info from request response data.
Storing them into three sub keys: {trace, func, data}.
Example:
{
"schema":"id_mapping",
"trace":{"funcID":1, "dataID":1, "children":[]},
"func":{ "<id>":{}},
"data":{ "<id>":{} }
}
The data part can be individually marshaled, upon error of a specific object, it's data can be ignored, and show an error prompt in the box.
from xgo.
Few possible optimizations:
- error silent when marshaling results and objects (done)
- set a size limit, for example 4K, which will make marshaled data shrinked when exceeding that limit (done)
- limit the depth of the stack
- limit the appearance of a specific function
from xgo.
This is an example trace, which is the count of calls for each function inside each package. It is large enough to cause the browser tab to crash (original file size 116M):
stats.json
Highlight:
{
"encoding/json": {
"newTypeEncoder": 72
},
"some.git.com/slc/c/framework_routinelocal/src/impl": {
"(*goroutineLocalImpl).Put": 31,
"(*goroutineLocalImpl).Value": 103
},
"some.git.com/slc/c/go_tls": {
"(*dataImpl).Value": 83,
"Get": 103,
"MakeData": 31,
"Set": 31,
"fetchDataMap": 134,
"getTlsData": 134
},
"some.git.com/slc/c/go_tls/g": {
"G": 134
},
"some.git.com/slc/upppb": {
"(*BizCommon).ProtoReflect": 77
},
"some.git.com/slc/upppc": {
"(*BizResult).ProtoReflect": 90
},
"github.com/golang/protobuf/proto": {
"String": 36
},
"github.com/prometheus/client_model/go": {
"(*LabelPair).GetName": 84
},
"google.golang.org/protobuf/encoding/protowire": {
"ConsumeBytes": 9927,
"ConsumeTag": 17035,
"ConsumeVarint": 34070,
"DecodeTag": 17035,
"EncodeTag": 68,
"SizeVarint": 68
},
"google.golang.org/protobuf/internal/filedesc": {
"(*Base).FullName": 3086,
"(*Base).Parent": 937,
"(*Base).Syntax": 61,
"(*Builder).optionsUnmarshaler": 3617,
"(*Enum).unmarshalFull": 112,
"(*EnumValue).unmarshalFull": 937,
"(*Field).Cardinality": 708,
"(*Field).ContainingOneof": 397,
"(*Field).HasPresence": 104,
"(*Field).IsList": 285,
"(*Field).IsMap": 651,
"(*Field).IsWeak": 139,
"(*Field).Kind": 479,
"(*Field).Message": 1201,
"(*Field).Number": 722,
"(*Field).unmarshalFull": 2051,
"(*Fields).Get": 622,
"(*Fields).Len": 684,
"(*File).Syntax": 61,
"(*File).lazyInit": 640,
"(*File).resolveMessageDependency": 615,
"(*Message).Fields": 582,
"(*Message).lazyInit": 638,
"(*Message).unmarshalFull": 416,
"(*stringName).InitJSON": 2051,
"appendFullName": 3084,
"makeFullName": 625
},
"google.golang.org/protobuf/internal/filetype": {
"(*resolverByIndex).FindMessageByIndex": 613,
"depIdxs.Get": 622
},
"google.golang.org/protobuf/internal/impl": {
"Export.protoMessageV2Of": 210,
"offsetOf": 148,
"pointer.Apply": 68,
"pointer.IsNil": 405,
"pointerOfIface": 265
},
"google.golang.org/protobuf/internal/strs": {
"(*Builder).AppendFullName": 3084,
"(*Builder).MakeString": 2683,
"(*Builder).grow": 5767,
"(*Builder).last": 5767,
"EnforceUTF8": 14,
"UnsafeString": 8851
},
"google.golang.org/protobuf/reflect/protoreflect": {
"Value.IsValid": 2051
}
}
We can see there are too much unrelated function calls. Thus we can filter out items with appearance more than 100 times by default
from xgo.
After applying the appearance limited shrinking, the size of the original JSON reduced from 116M to 7.3M, and the browser now can handle it.
And it seems the majority of the stack trace is kept. No significant information is lost.
from xgo.
Related Issues (20)
- Proposal: xgo tool test-explorer HOT 5
- rename the standalone tool name to `go-coverage-tool` later. HOT 1
- xgo failed to run tests on Windows HOT 14
- Bug: --strace with vendor mode, dep check always failed
- Allow more stdlib functions to be mocked HOT 4
- Trace: extend default shrink size from 4K(4096) to 16K(16384)
- Coverage: make incremental coverage work on windows
- Feature: extend external dependency auto importing
- Bug: common_string.go:66:6: __xgo_orig__ redeclared in this block
- Coverage: support coverage from URL
- CI: Run tests for kubernetes and pocketbase nightly
- Feature: add Debug button to test explorer HOT 1
- Enhancement: dynamically show subcases running status in test explorer
- Bug: linux: copy_file_range: is a directory HOT 1
- Doc: add individual doc for trace, coverage and test-explorer
- Bug: build pocketbase failed: github.com/pocketbase/pocketbase/models/schema internal compiler error: panic: interface conversion: interface {} is string, not syntax.writeError HOT 3
- Doc: add doc for xgo's github workflow and git hooks
- Mock: make --trap-stdlib the default when running `xgo test`
- Perf: Test-Explorer: Join all test files together to run all in once
- Proposal: use text files to patch runtime and compiler
Recommend Projects
-
React
A declarative, efficient, and flexible JavaScript library for building user interfaces.
-
Vue.js
🖖 Vue.js is a progressive, incrementally-adoptable JavaScript framework for building UI on the web.
-
Typescript
TypeScript is a superset of JavaScript that compiles to clean JavaScript output.
-
TensorFlow
An Open Source Machine Learning Framework for Everyone
-
Django
The Web framework for perfectionists with deadlines.
-
Laravel
A PHP framework for web artisans
-
D3
Bring data to life with SVG, Canvas and HTML. 📊📈🎉
-
Recommend Topics
-
javascript
JavaScript (JS) is a lightweight interpreted programming language with first-class functions.
-
web
Some thing interesting about web. New door for the world.
-
server
A server is a program made to process requests and deliver data to clients.
-
Machine learning
Machine learning is a way of modeling and interpreting data that allows a piece of software to respond intelligently.
-
Visualization
Some thing interesting about visualization, use data art
-
Game
Some thing interesting about game, make everyone happy.
Recommend Org
-
Facebook
We are working to build community through open source technology. NB: members must have two-factor auth.
-
Microsoft
Open source projects and samples from Microsoft.
-
Google
Google ❤️ Open Source for everyone.
-
Alibaba
Alibaba Open Source for everyone
-
D3
Data-Driven Documents codes.
-
Tencent
China tencent open source team.
from xgo.