// Narrow integer-column storage (task 213): values that fit a small byte // class after a minimum offset store little-endian deltas at width 1, 2, // 4, or 6 instead of 8 bytes each. Narrowing is a storage choice made at // column build -- the logical Dtype stays I64 and values read back // identically through every class. The 6-byte class exists for // epoch-millis timestamp columns, whose offset spans sit at 37-45 bits -- // past u32, far under u64. Dense, sparse, and rank layouts share one // value-vector representation.
package chickpeas import ( ) // narrowI64Vec is the shared narrow value store: value i = min + delta_i. type narrowI64Vec struct { min int64 w uint8 b []byte } // n is the element count. func ( narrowI64Vec) () int { return len(.b) / int(.w) } // at is the decoded value at slot i (caller bounds-checks). The u48 // class reads one unaligned 8-byte load masked to 48 bits -- the buffer // carries a 2-byte pad so the load never runs off the end; two narrow // loads plus a shift-combine measurably taxed per-candidate reads in // microsecond-scale kernels (task 300). func ( narrowI64Vec) ( int) int64 { switch .w { case 1: return .min + int64(.b[]) case 2: return .min + int64(binary.LittleEndian.Uint16(.b[*2:])) case 4: return .min + int64(binary.LittleEndian.Uint32(.b[*4:])) } return .min + int64(binary.LittleEndian.Uint64(.b[*6:])&narrowU48Mask) } // narrowU48Mask keeps the low 48 bits of the padded 8-byte u48 load. const narrowU48Mask = 0xFFFFFFFFFFFF // narrowI64MinLen gates narrowing to columns big enough for the byte // savings to matter; below it the extra representation buys nothing. const narrowI64MinLen = 1024 // narrowI64Vals encodes vals into the narrowest byte class whose offset // span fits; ok is false when the column is too small to bother or the // span needs a full 8 bytes (ids and similar full-range values). func narrowI64Vals( []int64) (narrowI64Vec, bool) { if len() < narrowI64MinLen { return narrowI64Vec{}, false } , := [0], [0] for , := range { if < { = } if > { = } } := uint64( - ) var uint8 switch { case <= 0xFF: = 1 case <= 0xFFFF: = 2 case <= 0xFFFFFFFF: = 4 case <= 0xFFFFFFFFFFFF: = 6 default: return narrowI64Vec{}, false } // The u48 buffer carries a 2-byte pad so at() can decode with one // 8-byte load; n() = len/w is unaffected (2 < w) and the pad is // never addressed as an element. := 0 if == 6 { = 2 } := make([]byte, len()*int()+) for , := range { := uint64( - ) switch { case 1: [] = byte() case 2: binary.LittleEndian.PutUint16([*2:], uint16()) case 4: binary.LittleEndian.PutUint32([*4:], uint32()) default: binary.LittleEndian.PutUint32([*6:], uint32()) binary.LittleEndian.PutUint16([*6+4:], uint16(>>32)) } } return narrowI64Vec{min: , w: , b: }, true } // denseI64NarrowCol is the dense narrow layout: slot == position. type denseI64NarrowCol struct { narrowI64Vec } // at keeps the historical position-typed accessor. func ( denseI64NarrowCol) ( uint32) int64 { return .narrowI64Vec.at(int()) } func ( denseI64NarrowCol) ( uint32) (Value, bool) { if int() >= .n() { return Value{}, false } return I64Value(.at()), true } func ( denseI64NarrowCol) () iter.Seq2[uint32, Value] { return func( func(uint32, Value) bool) { for := range .n() { if !(uint32(), I64Value(.narrowI64Vec.at())) { return } } } } func ( denseI64NarrowCol) () Dtype { return DtypeI64 } func ( denseI64NarrowCol) () int { return .n() } // denseI64BitCol is the two-valued dense class: a column whose values are // all min or min+1 (0/1 flags stored as integers being the archetype) // packs to one bit per position. Dtype stays I64 like every narrow class. type denseI64BitCol struct { min int64 bits *bitset.Bits } func ( denseI64BitCol) ( uint32) int64 { if .bits.Get(int()) { return .min + 1 } return .min } func ( denseI64BitCol) ( uint32) (Value, bool) { if int() >= .bits.Len() { return Value{}, false } return I64Value(.at()), true } func ( denseI64BitCol) () iter.Seq2[uint32, Value] { return func( func(uint32, Value) bool) { for := range .bits.Len() { if !(uint32(), I64Value(.at(uint32()))) { return } } } } func ( denseI64BitCol) () Dtype { return DtypeI64 } func ( denseI64BitCol) () int { return .bits.Len() } // narrowI64Column picks the dense storage class: one bit per position for // a two-valued column, else the narrowest byte class the span allows, // else plain []int64. func narrowI64Column( []int64) Column { if len() >= narrowI64MinLen { , := [0], [0] for , := range { if < { = } if > { = } } if uint64(-) <= 1 { := bitset.New(len()) for , := range { .Set(, != ) } return denseI64BitCol{min: , bits: } } } if , := narrowI64Vals(); { return denseI64NarrowCol{} } return denseI64Col() } // sparseI64NarrowCol is the sparse narrow layout: position-sorted ids with // narrow values, binary-searched exactly like sparseI64Col. type sparseI64NarrowCol struct { ids []uint32 narrowI64Vec } func ( sparseI64NarrowCol) ( uint32) (Value, bool) { if , := sparseSlot(.ids, ); { return I64Value(.at()), true } return Value{}, false } func ( sparseI64NarrowCol) () iter.Seq2[uint32, Value] { return func( func(uint32, Value) bool) { for , := range .ids { if !(, I64Value(.at())) { return } } } } func ( sparseI64NarrowCol) () Dtype { return DtypeI64 } func ( sparseI64NarrowCol) () int { return len(.ids) } // rankI64NarrowCol is the rank-select narrow layout. type rankI64NarrowCol struct { rankIndex narrowI64Vec } func ( rankI64NarrowCol) ( uint32) (Value, bool) { if , := .slot(); { return I64Value(.at()), true } return Value{}, false } func ( rankI64NarrowCol) () iter.Seq2[uint32, Value] { return .entries(func( int) Value { return I64Value(.at()) }) } func ( rankI64NarrowCol) () Dtype { return DtypeI64 } func ( rankI64NarrowCol) () int { return .n() }