// Licensed to the Apache Software Foundation (ASF) under one// or more contributor license agreements. See the NOTICE file// distributed with this work for additional information// regarding copyright ownership. The ASF licenses this file// to you under the Apache License, Version 2.0 (the// "License"); you may not use this file except in compliance// with the License. You may obtain a copy of the License at//// http://www.apache.org/licenses/LICENSE-2.0//// Unless required by applicable law or agreed to in writing, software// distributed under the License is distributed on an "AS IS" BASIS,// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.// See the License for the specific language governing permissions and// limitations under the License.package ipcimport ()const ( errNotArrowFile = errString("arrow/ipc: not an Arrow file") errInconsistentFileMetadata = errString("arrow/ipc: file is smaller than indicated metadata size") errInconsistentSchema = errString("arrow/ipc: tried to write record batch with different schema") errMaxRecursion = errString("arrow/ipc: max recursion depth reached") errBigArray = errString("arrow/ipc: array larger than 2^31-1 in length") kArrowAlignment = 64// buffers are padded to 64b boundaries (for SIMD) kTensorAlignment = 64// tensors are padded to 64b boundaries kArrowIPCAlignment = 8// align on 8b boundaries in IPC)var ( paddingBytes [kArrowAlignment]byte kEOS = [8]byte{0xFF, 0xFF, 0xFF, 0xFF, 0, 0, 0, 0} // end of stream message kIPCContToken uint32 = 0xFFFFFFFF// 32b continuation indicator for FlatBuffers 8b alignment)func paddedLength( int64, int32) int64 { := int64()return (( + - 1) / ) * }type errString stringfunc ( errString) () string {returnstring()}typeReadAtSeekerinterface {io.Readerio.Seekerio.ReaderAt}type config struct { alloc memory.Allocator schema *arrow.Schema footer struct { offset int64 } codec flatbuf.CompressionType compressNP int ensureNativeEndian bool noAutoSchema bool emitDictDeltas bool minSpaceSavings *float64}func newConfig( ...Option) *config { := &config{alloc: memory.NewGoAllocator(),codec: -1, // uncompressedensureNativeEndian: true,compressNP: 1, }for , := range { () }return}// Option is a functional option to configure opening or creating Arrow files// and streams.typeOptionfunc(*config)// WithFooterOffset specifies the Arrow footer position in bytes.func ( int64) Option {returnfunc( *config) { .footer.offset = }}// WithAllocator specifies the Arrow memory allocator used while building records.func ( memory.Allocator) Option {returnfunc( *config) { .alloc = }}// WithSchema specifies the Arrow schema to be used for reading or writing.func ( *arrow.Schema) Option {returnfunc( *config) { .schema = }}// WithLZ4 tells the writer to use LZ4 Frame compression on the data// buffers before writing. Requires >= Arrow 1.0.0 to read/decompressfunc () Option {returnfunc( *config) { .codec = flatbuf.CompressionTypeLZ4_FRAME }}// WithZstd tells the writer to use ZSTD compression on the data// buffers before writing. Requires >= Arrow 1.0.0 to read/decompressfunc () Option {returnfunc( *config) { .codec = flatbuf.CompressionTypeZSTD }}// WithCompressConcurrency specifies a number of goroutines to spin up for// concurrent compression of the body buffers when writing compress IPC records.// If n <= 1 then compression will be done serially without goroutine// parallelization. Default is 1.func ( int) Option {returnfunc( *config) {if <= 0 { = 1 } .compressNP = }}// WithEnsureNativeEndian specifies whether or not to automatically byte-swap// buffers with endian-sensitive data if the schema's endianness is not the// platform-native endianness. This includes all numeric types, temporal types,// decimal types, as well as the offset buffers of variable-sized binary and// list-like types.//// This is only relevant to ipc Reader objects, not to writers. This defaults// to true.func ( bool) Option {returnfunc( *config) { .ensureNativeEndian = }}// WithDelayedReadSchema alters the ipc.Reader behavior to delay attempting// to read the schema from the stream until the first call to Next instead// of immediately attempting to read a schema from the stream when created.func ( bool) Option {returnfunc( *config) { .noAutoSchema = }}// WithDictionaryDeltas specifies whether or not to emit dictionary deltas.func ( bool) Option {returnfunc( *config) { .emitDictDeltas = }}// WithMinSpaceSavings specifies a percentage of space savings for// compression to be applied to buffers.//// Space savings is calculated as (1.0 - compressedSize / uncompressedSize).//// For example, if minSpaceSavings = 0.1, a 100-byte body buffer won't// undergo compression if its expected compressed size exceeds 90 bytes.// If this option is unset, compression will be used indiscriminately. If// no codec was supplied, this option is ignored.//// Values outside of the range [0,1] are handled as errors.//// Note that enabling this option may result in unreadable data for Arrow// Go and C++ versions prior to 12.0.0.func ( float64) Option {returnfunc( *config) { .minSpaceSavings = & }}var ( _ arrio.Reader = (*Reader)(nil) _ arrio.Writer = (*Writer)(nil) _ arrio.Reader = (*FileReader)(nil) _ arrio.Writer = (*FileWriter)(nil) _ arrio.ReaderAt = (*FileReader)(nil))
The pages are generated with Goldsv0.8.2. (GOOS=linux GOARCH=amd64)
Golds is a Go 101 project developed by Tapir Liu.
PR and bug reports are welcome and can be submitted to the issue list.
Please follow @zigo_101 (reachable from the left QR code) to get the latest news of Golds.