// Package buffer implements a buffer for serialization, consisting of a chain of []byte-s to // reduce copying and to allow reuse of individual chunks.
package buffer import ( ) // PoolConfig contains configuration for the allocation and reuse strategy. type PoolConfig struct { StartSize int // Minimum chunk size that is allocated. PooledSize int // Minimum chunk size that is reused, reusing chunks too small will result in overhead. MaxSize int // Maximum chunk size that will be allocated. } var config = PoolConfig{ StartSize: 128, PooledSize: 512, MaxSize: 32768, } // Reuse pool: chunk size -> pool. var buffers = map[int]*sync.Pool{} func initBuffers() { for := config.PooledSize; <= config.MaxSize; *= 2 { buffers[] = new(sync.Pool) } } func init() { initBuffers() } // Init sets up a non-default pooling and allocation strategy. Should be run before serialization is done. func ( PoolConfig) { config = initBuffers() } // putBuf puts a chunk to reuse pool if it can be reused. func putBuf( []byte) { := cap() if < config.PooledSize { return } if := buffers[]; != nil { .Put([:0]) } } // getBuf gets a chunk from reuse pool or creates a new one if reuse failed. func getBuf( int) []byte { if >= config.PooledSize { if := buffers[]; != nil { := .Get() if != nil { return .([]byte) } } } return make([]byte, 0, ) } // Buffer is a buffer optimized for serialization without extra copying. type Buffer struct { // Buf is the current chunk that can be used for serialization. Buf []byte toPool []byte bufs [][]byte } // EnsureSpace makes sure that the current chunk contains at least s free bytes, // possibly creating a new chunk. func ( *Buffer) ( int) { if cap(.Buf)-len(.Buf) < { .ensureSpaceSlow() } } func ( *Buffer) ( int) { := len(.Buf) if > 0 { if cap(.toPool) != cap(.Buf) { // Chunk was reallocated, toPool can be pooled. putBuf(.toPool) } if cap(.bufs) == 0 { .bufs = make([][]byte, 0, 8) } .bufs = append(.bufs, .Buf) = cap(.toPool) * 2 } else { = config.StartSize } if > config.MaxSize { = config.MaxSize } .Buf = getBuf() .toPool = .Buf } // AppendByte appends a single byte to buffer. func ( *Buffer) ( byte) { .EnsureSpace(1) .Buf = append(.Buf, ) } // AppendBytes appends a byte slice to buffer. func ( *Buffer) ( []byte) { if len() <= cap(.Buf)-len(.Buf) { .Buf = append(.Buf, ...) // fast path } else { .appendBytesSlow() } } func ( *Buffer) ( []byte) { for len() > 0 { .EnsureSpace(1) := cap(.Buf) - len(.Buf) if > len() { = len() } .Buf = append(.Buf, [:]...) = [:] } } // AppendString appends a string to buffer. func ( *Buffer) ( string) { if len() <= cap(.Buf)-len(.Buf) { .Buf = append(.Buf, ...) // fast path } else { .appendStringSlow() } } func ( *Buffer) ( string) { for len() > 0 { .EnsureSpace(1) := cap(.Buf) - len(.Buf) if > len() { = len() } .Buf = append(.Buf, [:]...) = [:] } } // Size computes the size of a buffer by adding sizes of every chunk. func ( *Buffer) () int { := len(.Buf) for , := range .bufs { += len() } return } // DumpTo outputs the contents of a buffer to a writer and resets the buffer. func ( *Buffer) ( io.Writer) ( int, error) { := net.Buffers(.bufs) if len(.Buf) > 0 { = append(, .Buf) } , := .WriteTo() for , := range .bufs { putBuf() } putBuf(.toPool) .bufs = nil .Buf = nil .toPool = nil return int(), } // BuildBytes creates a single byte slice with all the contents of the buffer. Data is // copied if it does not fit in a single chunk. You can optionally provide one byte // slice as argument that it will try to reuse. func ( *Buffer) ( ...[]byte) []byte { if len(.bufs) == 0 { := .Buf .toPool = nil .Buf = nil return } var []byte := .Size() // If we got a buffer as argument and it is big enough, reuse it. if len() == 1 && cap([0]) >= { = [0][:0] } else { = make([]byte, 0, ) } for , := range .bufs { = append(, ...) putBuf() } = append(, .Buf...) putBuf(.toPool) .bufs = nil .toPool = nil .Buf = nil return } type readCloser struct { offset int bufs [][]byte } func ( *readCloser) ( []byte) ( int, error) { for , := range .bufs { // Copy as much as we can. := copy([:], [.offset:]) += // Increment how much we filled. // Did we empty the whole buffer? if .offset+ == len() { // On to the next buffer. .offset = 0 .bufs = .bufs[1:] // We can release this buffer. putBuf() } else { .offset += } if == len() { break } } // No buffers left or nothing read? if len(.bufs) == 0 { = io.EOF } return } func ( *readCloser) () error { // Release all remaining buffers. for , := range .bufs { putBuf() } // In case Close gets called multiple times. .bufs = nil return nil } // ReadCloser creates an io.ReadCloser with all the contents of the buffer. func ( *Buffer) () io.ReadCloser { := &readCloser{0, append(.bufs, .Buf)} .bufs = nil .toPool = nil .Buf = nil return }