Source File
	rcpg.go

Belonging Package
	github.com/freeeve/gochickpeas/rcpg

// Package rcpg reads and writes RCPG, the RustyChickpeas graph file, and its
// companion RRSR record store (subpackage rrsr). The byte layout is frozen by
// FORMAT.md in the rustychickpeas repository; this implementation is
// conformance-tested byte-for-byte against the Rust codec.
//
// Quick sketch (all integers little-endian):
//
//	header (16 B):  magic "RCPG" | version u16 | flags u16 | section_count u32 | reserved u32
//	directory:      section_count x { id u32, reserved u32, offset u64, length u64 }
//	sections:       at the directory offsets (relative to file start)
//
// Section IDs: 1 atoms, 2 meta, 3 nodes, 4 relationships, 5 node columns,
// 6 relationship columns, 7 optional atom block index, 8 optional node
// existence bitmap (sparse id spaces only -- absent means every
// in-id-space id is a node, which is exactly true for dense writers).
// Unknown section IDs are ignored on read so the format can grow; files
// with a version above 1 are rejected.
package rcpg

import (
	"bytes"
	"io"
)

// Magic is the RCPG file magic.
var Magic = [4]byte{'R', 'C', 'P', 'G'}

// Version is the RCPG format version this package reads and writes.
const Version uint16 = 1

const (
	sectionAtoms     uint32 = 1
	sectionMeta      uint32 = 2
	sectionNodes     uint32 = 3
	sectionRels      uint32 = 4
	sectionNodeCols  uint32 = 5
	sectionRelCols   uint32 = 6
	sectionAtomIndex uint32 = 7
	sectionExistence uint32 = 8
)

// atomIndexBlockLen is the writer default for the section-7 atom index's
// atoms-per-block: ~4 KB of u64 offsets per million atoms, small enough to
// hold resident on a lazy reader.
const atomIndexBlockLen = 1024

const (
	headerLen   = 16
	dirEntryLen = 24
)

// WriteOptions controls which optional sections a writer emits. Topology
// (atoms, meta, nodes, relationships) is always written; property columns
// are optional so large graphs can ship a lean traversal-only file and leave
// per-node data to a range-fetched record store.
type WriteOptions struct {
	NodeColumns bool
	RelColumns  bool
	// AtomIndex emits the optional section-7 atom block index. Default
	// off, so existing byte-identity guarantees (and the vendored
	// conformance corpus) are unchanged unless a writer opts in.
	AtomIndex bool
	// Existence emits the optional section-8 node existence bitmap when
	// the graph carries one (sparse id spaces). Default off for the same
	// byte-identity reason as AtomIndex: the conformance corpus is the
	// cross-engine contract, and the section joins it in a coordinated
	// corpus bump once the Rust writer emits it too. Until a sparse
	// graph is written WITH this, its round trip keeps the legacy
	// every-in-space-id-exists presumption.
	Existence bool
}

// DefaultWriteOptions emits every section.
func DefaultWriteOptions() WriteOptions {
	return WriteOptions{NodeColumns: true, RelColumns: true}
}

// TopologyOnlyWriteOptions emits no property column sections.
func TopologyOnlyWriteOptions() WriteOptions {
	return WriteOptions{}
}

// ParseOptions controls which optional sections a reader materializes.
// Skipped sections cost nothing to parse and nothing in resident memory,
// regardless of whether the file contains them.
type ParseOptions struct {
	NodeColumns bool
	RelColumns  bool
	// SkipAtoms leaves the atom table out of the parse: GraphSection.Atoms
	// stays nil and ParseLazy never fetches the atoms section. Labels, rel
	// types, and property keys remain raw atom ids; resolve them on demand
	// through an AtomReader over the same SectionFetch.
	SkipAtoms bool
}

// DefaultParseOptions materializes every section.
func DefaultParseOptions() ParseOptions {
	return ParseOptions{NodeColumns: true, RelColumns: true}
}

// TopologyOnlyParseOptions ignores property column sections even when
// present.
func TopologyOnlyParseOptions() ParseOptions {
	return ParseOptions{}
}

// SkeletonParseOptions materializes the CSR skeleton only: no property
// columns and no atom table. The lightest resident load for traversal-first
// consumers that resolve atom ids lazily via AtomReader.
func SkeletonParseOptions() ParseOptions {
	return ParseOptions{SkipAtoms: true}
}

// Write serializes a graph to RCPG bytes, including all sections.
func Write(g *GraphSection, w io.Writer) error {
	return WriteWith(g, w, DefaultWriteOptions())
}

// WriteWith serializes a graph to RCPG bytes, emitting optional sections per
// opts. The output is deterministic: section order, and the order of bitmap
// index entries and columns within their sections, follow the GraphSection
// slices exactly (which the spec requires sorted by atom).
func WriteWith(g *GraphSection, w io.Writer, opts WriteOptions) error {
	type section struct {
		id   uint32
		body []byte
	}
	encode := func(id uint32, f func() ([]byte, error)) (section, error) {
		body, err := f()
		return section{id: id, body: body}, err
	}
	sections := make([]section, 0, 6)
	for _, enc := range []struct {
		id      uint32
		f       func() ([]byte, error)
		enabled bool
	}{
		{sectionAtoms, func() ([]byte, error) { return encodeAtoms(g.Atoms) }, true},
		{sectionMeta, func() ([]byte, error) { return encodeMeta(g) }, true},
		{sectionNodes, func() ([]byte, error) { return encodeNodes(g) }, true},
		{sectionRels, func() ([]byte, error) { return encodeRels(g) }, true},
		{sectionNodeCols, func() ([]byte, error) { return encodeColumns(g.NodeColumns) }, opts.NodeColumns},
		{sectionRelCols, func() ([]byte, error) { return encodeColumns(g.RelColumns) }, opts.RelColumns},
		{sectionAtomIndex, func() ([]byte, error) { return encodeAtomIndex(g.Atoms, atomIndexBlockLen) }, opts.AtomIndex},
		{sectionExistence, func() ([]byte, error) { return encodeExistence(g.Existence) }, opts.Existence && g.Existence != nil},
	} {
		if !enc.enabled {
			continue
		}
		s, err := encode(enc.id, enc.f)
		if err != nil {
			return err
		}
		sections = append(sections, s)
	}

	var head bytes.Buffer
	head.Write(Magic[:])
	wU16(&head, Version)
	wU16(&head, 0) // flags
	wU32(&head, uint32(len(sections)))
	wU32(&head, 0) // reserved
	offset := uint64(headerLen + len(sections)*dirEntryLen)
	for _, s := range sections {
		wU32(&head, s.id)
		wU32(&head, 0) // reserved
		wU64(&head, offset)
		wU64(&head, uint64(len(s.body)))
		offset += uint64(len(s.body))
	}
	if _, err := w.Write(head.Bytes()); err != nil {
		return err
	}
	for _, s := range sections {
		if _, err := w.Write(s.body); err != nil {
			return err
		}
	}
	return nil
}

// Parse decodes RCPG bytes into a GraphSection, materializing all sections.
// It never panics on malformed input; all structural problems surface as
// errors wrapping ErrCorrupt (or ErrUnsupportedVersion for a future format
// version).
func Parse(b []byte) (*GraphSection, error) {
	return ParseWith(b, DefaultParseOptions())
}

// ParseWith decodes RCPG bytes, materializing optional sections per opts.
// Use TopologyOnlyParseOptions to keep property columns out of memory on
// large graphs.
func ParseWith(b []byte, opts ParseOptions) (*GraphSection, error) {
	dir, err := ParseDirectory(b)
	if err != nil {
		return nil, err
	}
	g := &GraphSection{}
	for _, e := range dir.Entries() {
		end := e.Offset + e.Length
		if end < e.Offset {
			return nil, corruptf("section %d offset+length overflows", e.ID)
		}
		if end > uint64(len(b)) {
			return nil, corruptf("section %d extends past end of file (%d > %d)",
				e.ID, end, len(b))
		}
		if err := decodeSection(e.ID, b[e.Offset:end], opts, g); err != nil {
			return nil, err
		}
	}
	// A PRESENT atom index must be correct -- a malformed one is corrupt,
	// never a silent fall-back to the scan path. Deep validation needs the
	// materialized atoms; a SkipAtoms parse keeps the structurally-decoded
	// index and leaves the per-block check to the consumer (AtomReader).
	if g.AtomIndex != nil && !opts.SkipAtoms {
		if err := validateAtomIndex(g.AtomIndex, g.Atoms); err != nil {
			return nil, err
		}
	}
	return g, nil
}

// decodeSection decodes one section body into g, honouring opts for the
// optional column sections. Unknown section IDs are ignored (forward
// compatibility). Shared by the eager ParseWith and the lazy ParseLazy paths
// so both decode identically.
func decodeSection(id uint32, body []byte, opts ParseOptions, g *GraphSection) error {
	var err error
	switch {
	case id == sectionAtoms && !opts.SkipAtoms:
		g.Atoms, err = decodeAtoms(body)
	case id == sectionMeta:
		err = decodeMeta(body, g)
	case id == sectionNodes:
		err = decodeNodes(body, g)
	case id == sectionRels:
		err = decodeRels(body, g)
	case id == sectionNodeCols && opts.NodeColumns:
		g.NodeColumns, err = decodeColumns(body)
	case id == sectionRelCols && opts.RelColumns:
		g.RelColumns, err = decodeColumns(body)
	case id == sectionAtomIndex:
		g.AtomIndex, err = decodeAtomIndex(body)
	case id == sectionExistence:
		g.Existence, err = decodeExistence(body)
	default:
		// Present-but-skipped columns, or forward compatibility: ignore
		// unknown sections.
	}
	return err
}

// sectionWanted reports whether a section is needed under opts: topology
// (meta/nodes/relationships) always, atoms unless skipped, the property
// columns only when their option is set. Unknown sections are never fetched.
func sectionWanted(id uint32, opts ParseOptions) bool {
	switch id {
	case sectionAtoms:
		return !opts.SkipAtoms
	case sectionMeta, sectionNodes, sectionRels, sectionAtomIndex:
		return true
	case sectionNodeCols:
		return opts.NodeColumns
	case sectionRelCols:
		return opts.RelColumns
	}
	return false
}

The pages are generated with Golds v0.8.7. (GOOS=linux GOARCH=amd64)
Golds is a Go 101 project developed by Tapir Liu.
PR and bug reports are welcome and can be submitted to the issue list.
Please follow @zigo_101 (reachable from the left QR code) to get the latest news of Golds.