X-Git-Url: https://git.giacomolli.uk/onechat/blobdiff_plain/baf7a940be8927278e3b4b87e7366cebfc16532f..c781f79f3c8cb9a7577ee1574d196988da2719b3:/tree.go?ds=sidebyside diff --git a/tree.go b/tree.go index af164ae..ad30eeb 100644 --- a/tree.go +++ b/tree.go @@ -23,6 +23,44 @@ const nodeBytes = 512 // the node line, the node index, and the node level. const treeRecordBytes = 32 +// Tree holds the built nodes of one chat. It writes one JSON line for each node +// to tree/YYYY-MM-DD.jsonl, named for the local day of the build. Each node is +// built once and never again. A node of a higher level is built only after both +// of its children. After a failed write, the tree refuses all writes until it +// is opened again. The caller holds the lock of the chat and serializes all +// calls. +// +// The tree keeps no node in memory. The index index/tree.idx lists the node +// lines in file order, and index/tree-L.idx holds the position of node (L, i) +// in record i, with a zero record for a node that is not built. At open the +// tree indexes the lines after the last indexed one, so a missing index is +// built again from the day files. +type Tree struct { + clock func() time.Time + journal Journal + dir string + order *Records + levels map[int]*Records + failure error +} + +// Node is one line of the summary tree. Node (l, i) covers the 2^l messages +// from i*2^l on. A node of level 0 holds one message. A node of a higher level +// merges its two children, (l-1, 2i) and (l-1, 2i+1). Size is the UTF-8 byte +// length of Text. +type Node struct { + Level int `json:"l"` + Index int `json:"i"` + Text string `json:"text"` + Size int `json:"size"` +} + +// NodeKey names the node (l, i). +type NodeKey struct { + level int + index int +} + // openTree opens the tree of the chat in root and indexes its new lines. A chat // of total messages cannot have a node that covers a later message. func openTree(ctx context.Context, root string, total int, clock func() time.Time) (*Tree, error) { @@ -50,70 +88,6 @@ func openTree(ctx context.Context, root string, total int, clock func() time.Tim return t, nil } -// leafSource returns the text that the leaf of message holds or summarizes. -func leafSource(message Message) string { - return message.Kind + ": " + message.Text -} - -// freeLeaf returns the leaf of message when its source fits in a node. Such a -// leaf holds the source word for word and needs no model call. -func freeLeaf(message Message) (Node, bool) { - text := leafSource(message) - if len(text) > nodeBytes { - return Node{}, false - } - - return Node{ - Level: 0, - Index: message.ID, - Text: text, - Size: len(text), - }, true -} - -// freeMerge returns the parent of two sibling nodes when both texts, joined by -// a newline, fit in a node. Such a parent needs no model call. -func freeMerge(left, right Node) (Node, bool) { - if left.Size+1+right.Size > nodeBytes { - return Node{}, false - } - - text := left.Text + "\n" + right.Text - - return Node{ - Level: left.Level + 1, - Index: left.Index / 2, - Text: text, - Size: len(text), - }, true -} - -// nodeEnd returns the ID after the last message of node (l, i). -func nodeEnd(level, index int) int { - return (index + 1) << level -} - -// Tree holds the built nodes of one chat. It writes one JSON line for each node -// to tree/YYYY-MM-DD.jsonl, named for the local day of the build. Each node is -// built once and never again. A node of a higher level is built only after both -// of its children. After a failed write, the tree refuses all writes until it -// is opened again. The caller holds the lock of the chat and serializes all -// calls. -// -// The tree keeps no node in memory. The index index/tree.idx lists the node -// lines in file order, and index/tree-L.idx holds the position of node (L, i) -// in record i, with a zero record for a node that is not built. At open the -// tree indexes the lines after the last indexed one, so a missing index is -// built again from the day files. -type Tree struct { - clock func() time.Time - journal Journal - dir string - order *Records - levels map[int]*Records - failure error -} - // Close closes the files of the tree. func (t *Tree) Close() error { err := t.journal.Close() @@ -124,6 +98,7 @@ func (t *Tree) Close() error { for level, index := range t.levels { err = errors.Join(err, index.Close()) + delete(t.levels, level) } @@ -178,6 +153,7 @@ func (t *Tree) Node(level, index int) (Node, bool, error) { } var node Node + err = json.Unmarshal(line, &node) if err != nil { return Node{}, false, fmt.Errorf("decode node (%d, %d): %w", level, index, err) @@ -212,6 +188,7 @@ func (t *Tree) Keys() ([]NodeKey, error) { func (t *Tree) open(ctx context.Context, total int) error { var err error + t.order, err = openRecords(filepath.Join(t.dir, "tree.idx"), treeRecordBytes) if err != nil { return err @@ -245,9 +222,10 @@ func (t *Tree) open(ctx context.Context, total int) error { return t.journal.Scan(ctx, start, func(line []byte, place Position) error { var node Node + err := json.Unmarshal(line, &node) if err != nil { - return err + return fmt.Errorf("decode tree node: %w", err) } err = t.check(node) @@ -389,21 +367,47 @@ func (t *Tree) Count() int { return t.order.count } -// Node is one line of the summary tree. Node (l, i) covers the 2^l messages -// from i*2^l on. A node of level 0 holds one message. A node of a higher level -// merges its two children, (l-1, 2i) and (l-1, 2i+1). Size is the UTF-8 byte -// length of Text. -type Node struct { - Level int `json:"l"` - Index int `json:"i"` - Text string `json:"text"` - Size int `json:"size"` +// leafSource returns the text that the leaf of message holds or summarizes. +func leafSource(message Message) string { + return message.Kind + ": " + message.Text } -// NodeKey names the node (l, i). -type NodeKey struct { - level int - index int +// freeLeaf returns a text-only leaf when its source fits in a node. Images need +// a model description even when their text fits. +func freeLeaf(message Message) (Node, bool) { + text := leafSource(message) + if len(text) > nodeBytes || len(imageFiles(message.Text)) > 0 { + return Node{}, false + } + + return Node{ + Level: 0, + Index: message.ID, + Text: text, + Size: len(text), + }, true +} + +// freeMerge returns the parent of two sibling nodes when both texts, joined by +// a newline, fit in a node. Such a parent needs no model call. +func freeMerge(left, right Node) (Node, bool) { + if left.Size+1+right.Size > nodeBytes { + return Node{}, false + } + + text := left.Text + "\n" + right.Text + + return Node{ + Level: left.Level + 1, + Index: left.Index / 2, + Text: text, + Size: len(text), + }, true +} + +// nodeEnd returns the ID after the last message of node (l, i). +func nodeEnd(level, index int) int { + return (index + 1) << level } // treeDecodeRecord reads the key and the position of a record of