// the node line, the node index, and the node level.
const treeRecordBytes = 32
+// Tree holds the built nodes of one chat. It writes one JSON line for each node
+// to tree/YYYY-MM-DD.jsonl, named for the local day of the build. Each node is
+// built once and never again. A node of a higher level is built only after both
+// of its children. After a failed write, the tree refuses all writes until it
+// is opened again. The caller holds the lock of the chat and serializes all
+// calls.
+//
+// The tree keeps no node in memory. The index index/tree.idx lists the node
+// lines in file order, and index/tree-L.idx holds the position of node (L, i)
+// in record i, with a zero record for a node that is not built. At open the
+// tree indexes the lines after the last indexed one, so a missing index is
+// built again from the day files.
+type Tree struct {
+ clock func() time.Time
+ journal Journal
+ dir string
+ order *Records
+ levels map[int]*Records
+ failure error
+}
+
+// Node is one line of the summary tree. Node (l, i) covers the 2^l messages
+// from i*2^l on. A node of level 0 holds one message. A node of a higher level
+// merges its two children, (l-1, 2i) and (l-1, 2i+1). Size is the UTF-8 byte
+// length of Text.
+type Node struct {
+ Level int `json:"l"`
+ Index int `json:"i"`
+ Text string `json:"text"`
+ Size int `json:"size"`
+}
+
+// NodeKey names the node (l, i).
+type NodeKey struct {
+ level int
+ index int
+}
+
// openTree opens the tree of the chat in root and indexes its new lines. A chat
// of total messages cannot have a node that covers a later message.
func openTree(ctx context.Context, root string, total int, clock func() time.Time) (*Tree, error) {
return t, nil
}
-// leafSource returns the text that the leaf of message holds or summarizes.
-func leafSource(message Message) string {
- return message.Kind + ": " + message.Text
-}
-
-// freeLeaf returns the leaf of message when its source fits in a node. Such a
-// leaf holds the source word for word and needs no model call.
-func freeLeaf(message Message) (Node, bool) {
- text := leafSource(message)
- if len(text) > nodeBytes {
- return Node{}, false
- }
-
- return Node{
- Level: 0,
- Index: message.ID,
- Text: text,
- Size: len(text),
- }, true
-}
-
-// freeMerge returns the parent of two sibling nodes when both texts, joined by
-// a newline, fit in a node. Such a parent needs no model call.
-func freeMerge(left, right Node) (Node, bool) {
- if left.Size+1+right.Size > nodeBytes {
- return Node{}, false
- }
-
- text := left.Text + "\n" + right.Text
-
- return Node{
- Level: left.Level + 1,
- Index: left.Index / 2,
- Text: text,
- Size: len(text),
- }, true
-}
-
-// nodeEnd returns the ID after the last message of node (l, i).
-func nodeEnd(level, index int) int {
- return (index + 1) << level
-}
-
-// Tree holds the built nodes of one chat. It writes one JSON line for each node
-// to tree/YYYY-MM-DD.jsonl, named for the local day of the build. Each node is
-// built once and never again. A node of a higher level is built only after both
-// of its children. After a failed write, the tree refuses all writes until it
-// is opened again. The caller holds the lock of the chat and serializes all
-// calls.
-//
-// The tree keeps no node in memory. The index index/tree.idx lists the node
-// lines in file order, and index/tree-L.idx holds the position of node (L, i)
-// in record i, with a zero record for a node that is not built. At open the
-// tree indexes the lines after the last indexed one, so a missing index is
-// built again from the day files.
-type Tree struct {
- clock func() time.Time
- journal Journal
- dir string
- order *Records
- levels map[int]*Records
- failure error
-}
-
// Close closes the files of the tree.
func (t *Tree) Close() error {
err := t.journal.Close()
for level, index := range t.levels {
err = errors.Join(err, index.Close())
+
delete(t.levels, level)
}
}
var node Node
+
err = json.Unmarshal(line, &node)
if err != nil {
return Node{}, false, fmt.Errorf("decode node (%d, %d): %w", level, index, err)
func (t *Tree) open(ctx context.Context, total int) error {
var err error
+
t.order, err = openRecords(filepath.Join(t.dir, "tree.idx"), treeRecordBytes)
if err != nil {
return err
return t.journal.Scan(ctx, start, func(line []byte, place Position) error {
var node Node
+
err := json.Unmarshal(line, &node)
if err != nil {
return fmt.Errorf("decode tree node: %w", err)
return t.order.count
}
-// Node is one line of the summary tree. Node (l, i) covers the 2^l messages
-// from i*2^l on. A node of level 0 holds one message. A node of a higher level
-// merges its two children, (l-1, 2i) and (l-1, 2i+1). Size is the UTF-8 byte
-// length of Text.
-type Node struct {
- Level int `json:"l"`
- Index int `json:"i"`
- Text string `json:"text"`
- Size int `json:"size"`
+// leafSource returns the text that the leaf of message holds or summarizes.
+func leafSource(message Message) string {
+ return message.Kind + ": " + message.Text
}
-// NodeKey names the node (l, i).
-type NodeKey struct {
- level int
- index int
+// freeLeaf returns a text-only leaf when its source fits in a node. Images need
+// a model description even when their text fits.
+func freeLeaf(message Message) (Node, bool) {
+ text := leafSource(message)
+ if len(text) > nodeBytes || len(imageFiles(message.Text)) > 0 {
+ return Node{}, false
+ }
+
+ return Node{
+ Level: 0,
+ Index: message.ID,
+ Text: text,
+ Size: len(text),
+ }, true
+}
+
+// freeMerge returns the parent of two sibling nodes when both texts, joined by
+// a newline, fit in a node. Such a parent needs no model call.
+func freeMerge(left, right Node) (Node, bool) {
+ if left.Size+1+right.Size > nodeBytes {
+ return Node{}, false
+ }
+
+ text := left.Text + "\n" + right.Text
+
+ return Node{
+ Level: left.Level + 1,
+ Index: left.Index / 2,
+ Text: text,
+ Size: len(text),
+ }, true
+}
+
+// nodeEnd returns the ID after the last message of node (l, i).
+func nodeEnd(level, index int) int {
+ return (index + 1) << level
}
// treeDecodeRecord reads the key and the position of a record of
projects / onechat / blobdiff