X-Git-Url: https://git.giacomolli.uk/onechat/blobdiff_plain/4343ca5fb6cc86eb887951f4fe16063bab29a238..98d137c3e945a09f76e85bd18238558c668fdca4:/tree.go?ds=inline

diff --git a/tree.go b/tree.go
index 079969c..765016e 100644
--- a/tree.go
+++ b/tree.go
@@ -23,76 +23,6 @@ const nodeBytes = 512
 // the node line, the node index, and the node level.
 const treeRecordBytes = 32
 
-// openTree opens the tree of the chat in root and indexes its new lines. A chat
-// of total messages cannot have a node that covers a later message.
-func openTree(ctx context.Context, root string, total int, clock func() time.Time) (*Tree, error) {
-	for _, dir := range []string{"tree", "index"} {
-		err := os.MkdirAll(filepath.Join(root, dir), 0o700)
-		if err != nil {
-			return nil, fmt.Errorf("create %s directory: %w", dir, err)
-		}
-	}
-
-	t := &Tree{
-		clock: clock,
-		journal: Journal{
-			dir: filepath.Join(root, "tree"),
-		},
-		dir:    filepath.Join(root, "index"),
-		levels: make(map[int]*Records),
-	}
-
-	err := t.open(ctx, total)
-	if err != nil {
-		return nil, errors.Join(fmt.Errorf("index tree: %w", err), t.Close())
-	}
-
-	return t, nil
-}
-
-// leafSource returns the text that the leaf of message holds or summarizes.
-func leafSource(message Message) string {
-	return message.Kind + ": " + message.Text
-}
-
-// freeLeaf returns a text-only leaf when its source fits in a node. Images need
-// a model description even when their text fits.
-func freeLeaf(message Message) (Node, bool) {
-	text := leafSource(message)
-	if len(text) > nodeBytes || len(imageFiles(message.Text)) > 0 {
-		return Node{}, false
-	}
-
-	return Node{
-		Level: 0,
-		Index: message.ID,
-		Text:  text,
-		Size:  len(text),
-	}, true
-}
-
-// freeMerge returns the parent of two sibling nodes when both texts, joined by
-// a newline, fit in a node. Such a parent needs no model call.
-func freeMerge(left, right Node) (Node, bool) {
-	if left.Size+1+right.Size > nodeBytes {
-		return Node{}, false
-	}
-
-	text := left.Text + "\n" + right.Text
-
-	return Node{
-		Level: left.Level + 1,
-		Index: left.Index / 2,
-		Text:  text,
-		Size:  len(text),
-	}, true
-}
-
-// nodeEnd returns the ID after the last message of node (l, i).
-func nodeEnd(level, index int) int {
-	return (index + 1) << level
-}
-
 // Tree holds the built nodes of one chat. It writes one JSON line for each node
 // to tree/YYYY-MM-DD.jsonl, named for the local day of the build. Each node is
 // built once and never again. A node of a higher level is built only after both
@@ -114,102 +44,48 @@ type Tree struct {
 	failure error
 }
 
-// Close closes the files of the tree.
-func (t *Tree) Close() error {
-	err := t.journal.Close()
-	if t.order != nil {
-		err = errors.Join(err, t.order.Close())
-		t.order = nil
-	}
-
-	for level, index := range t.levels {
-		err = errors.Join(err, index.Close())
-
-		delete(t.levels, level)
-	}
-
-	return err
-}
-
-// Add writes a new node to the tree.
-func (t *Tree) Add(node Node) error {
-	if t.failure != nil {
-		return fmt.Errorf("tree needs a reopen after a failed write: %w", t.failure)
-	}
-
-	err := t.check(node)
-	if err != nil {
-		return err
-	}
-
-	line, err := json.Marshal(node)
-	if err != nil {
-		return fmt.Errorf("encode node: %w", err)
-	}
-
-	place, err := t.journal.Write(t.clock().Format(time.DateOnly), append(line, '\n'))
-	if err == nil {
-		err = t.addIndex(NodeKey{node.Level, node.Index}, place)
-	}
-
-	if err != nil {
-		t.failure = err
-		return fmt.Errorf("write node (%d, %d): %w", node.Level, node.Index, err)
-	}
-
-	return nil
+// Node is one line of the summary tree. Node (l, i) covers the 2^l messages
+// from i*2^l on. A node of level 0 holds one message. A node of a higher level
+// merges its two children, (l-1, 2i) and (l-1, 2i+1). Size is the UTF-8 byte
+// length of Text.
+type Node struct {
+	Level int    `json:"l"`
+	Index int    `json:"i"`
+	Text  string `json:"text"`
+	Size  int    `json:"size"`
 }
 
-// Built tells if the node (l, i) is built.
-func (t *Tree) Built(level, index int) (bool, error) {
-	_, built, err := t.place(level, index)
-	return built, err
+// NodeKey names the node (l, i).
+type NodeKey struct {
+	level int
+	index int
 }
 
-// Node reads the node (l, i) if it is built.
-func (t *Tree) Node(level, index int) (Node, bool, error) {
-	place, built, err := t.place(level, index)
-	if err != nil || !built {
-		return Node{}, false, err
+// openTree opens the tree of the chat in root and indexes its new lines. A chat
+// of total messages cannot have a node that covers a later message.
+func openTree(ctx context.Context, root string, total int, clock func() time.Time) (*Tree, error) {
+	for _, dir := range []string{"tree", "index"} {
+		err := os.MkdirAll(filepath.Join(root, dir), 0o700)
+		if err != nil {
+			return nil, fmt.Errorf("create %s directory: %w", dir, err)
+		}
 	}
 
-	line, err := t.journal.Read(place)
-	if err != nil {
-		return Node{}, false, fmt.Errorf("read node (%d, %d): %w", level, index, err)
+	t := &Tree{
+		clock: clock,
+		journal: Journal{
+			dir: filepath.Join(root, "tree"),
+		},
+		dir:    filepath.Join(root, "index"),
+		levels: make(map[int]*Records),
 	}
 
-	var node Node
-
-	err = json.Unmarshal(line, &node)
+	err := t.open(ctx, total)
 	if err != nil {
-		return Node{}, false, fmt.Errorf("decode node (%d, %d): %w", level, index, err)
-	}
-
-	if node.Level != level || node.Index != index {
-		return Node{}, false, fmt.Errorf("index of node (%d, %d) points to node (%d, %d)", level, index, node.Level, node.Index)
-	}
-
-	return node, true, nil
-}
-
-// Keys returns the keys of the built nodes, ordered by level and then by index.
-func (t *Tree) Keys() ([]NodeKey, error) {
-	keys := make([]NodeKey, 0, t.order.count)
-	for n := range t.order.count {
-		record, _, err := t.order.Read(n)
-		if err != nil {
-			return nil, err
-		}
-
-		key, _ := treeDecodeRecord(record)
-		keys = append(keys, key)
+		return nil, errors.Join(fmt.Errorf("index tree: %w", err), t.Close())
 	}
 
-	slices.SortFunc(keys, func(a, b NodeKey) int {
-		return cmp.Or(cmp.Compare(a.level, b.level), cmp.Compare(a.index, b.index))
-	})
-
-	return keys, nil
+	return t, nil
 }
 
 func (t *Tree) open(ctx context.Context, total int) error {
@@ -217,19 +93,19 @@ func (t *Tree) open(ctx context.Context, total int) error {
 
 	t.order, err = openRecords(filepath.Join(t.dir, "tree.idx"), treeRecordBytes)
 	if err != nil {
-		return err
+		return fmt.Errorf("open node order index: %w", err)
 	}
 
 	start := Position{}
 	if t.order.count == 0 {
 		err = t.removeLevels()
 		if err != nil {
-			return err
+			return fmt.Errorf("remove empty level indexes: %w", err)
 		}
 	} else {
 		record, _, err := t.order.Read(t.order.count - 1)
 		if err != nil {
-			return err
+			return fmt.Errorf("read last node order: %w", err)
 		}
 
 		// A crash can stop an add between the two index writes, so the
@@ -237,7 +113,7 @@ func (t *Tree) open(ctx context.Context, total int) error {
 		node, last := treeDecodeRecord(record)
 		err = t.writeLevel(node, last)
 		if err != nil {
-			return err
+			return fmt.Errorf("restore last node level: %w", err)
 		}
 
 		start = Position{
@@ -246,7 +122,7 @@ func (t *Tree) open(ctx context.Context, total int) error {
 		}
 	}
 
-	return t.journal.Scan(ctx, start, func(line []byte, place Position) error {
+	err = t.journal.Scan(ctx, start, func(line []byte, place Position) error {
 		var node Node
 
 		err := json.Unmarshal(line, &node)
@@ -256,15 +132,78 @@ func (t *Tree) open(ctx context.Context, total int) error {
 
 		err = t.check(node)
 		if err != nil {
-			return err
+			return fmt.Errorf("validate scanned node: %w", err)
 		}
 
 		if nodeEnd(node.Level, node.Index) > total {
 			return fmt.Errorf("node (%d, %d) covers messages after the log", node.Level, node.Index)
 		}
 
-		return t.addIndex(NodeKey{node.Level, node.Index}, place)
+		err = t.addIndex(NodeKey{node.Level, node.Index}, place)
+		if err != nil {
+			return fmt.Errorf("index scanned node: %w", err)
+		}
+
+		return nil
 	})
+	if err != nil {
+		return fmt.Errorf("scan node journal: %w", err)
+	}
+
+	return nil
+}
+
+// Close closes the files of the tree.
+func (t *Tree) Close() error {
+	err := t.journal.Close()
+	if t.order != nil {
+		err = errors.Join(err, t.order.Close())
+		t.order = nil
+	}
+
+	for level, index := range t.levels {
+		err = errors.Join(err, index.Close())
+
+		delete(t.levels, level)
+	}
+
+	return err
+}
+
+// removeLevels removes the level indexes, which are not valid without the order
+// index.
+func (t *Tree) removeLevels() error {
+	paths, err := filepath.Glob(filepath.Join(t.dir, "tree-*.idx"))
+	if err != nil {
+		return fmt.Errorf("list level indexes: %w", err)
+	}
+
+	for _, path := range paths {
+		err = os.Remove(path)
+		if err != nil {
+			return fmt.Errorf("remove level index: %w", err)
+		}
+	}
+
+	return nil
+}
+
+// writeLevel writes the position of node to its level index.
+func (t *Tree) writeLevel(node NodeKey, place Position) error {
+	level, err := t.level(node.level)
+	if err != nil {
+		return fmt.Errorf("open node level index: %w", err)
+	}
+
+	record := make([]byte, positionBytes)
+	encodePosition(record, place)
+
+	err = level.Write(node.index, record)
+	if err != nil {
+		return fmt.Errorf("write node level record: %w", err)
+	}
+
+	return nil
 }
 
 // check makes sure that node is valid and can join the tree now.
@@ -279,7 +218,7 @@ func (t *Tree) check(node Node) error {
 
 	built, err := t.Built(node.Level, node.Index)
 	if err != nil {
-		return err
+		return fmt.Errorf("check target node: %w", err)
 	}
 
 	if built {
@@ -293,7 +232,7 @@ func (t *Tree) check(node Node) error {
 	for _, index := range []int{2 * node.Index, 2*node.Index + 1} {
 		built, err = t.Built(node.Level-1, index)
 		if err != nil {
-			return err
+			return fmt.Errorf("check child node: %w", err)
 		}
 
 		if !built {
@@ -313,10 +252,38 @@ func (t *Tree) addIndex(node NodeKey, place Position) error {
 
 	err := t.order.Append(record)
 	if err != nil {
-		return err
+		return fmt.Errorf("append node order: %w", err)
+	}
+
+	err = t.writeLevel(node, place)
+	if err != nil {
+		return fmt.Errorf("write indexed node level: %w", err)
+	}
+
+	return nil
+}
+
+// level returns the index of level, and opens it at first use.
+func (t *Tree) level(level int) (*Records, error) {
+	index, open := t.levels[level]
+	if open {
+		return index, nil
+	}
+
+	index, err := openRecords(filepath.Join(t.dir, fmt.Sprintf("tree-%d.idx", level)), positionBytes)
+	if err != nil {
+		return nil, err
 	}
 
-	return t.writeLevel(node, place)
+	t.levels[level] = index
+
+	return index, nil
+}
+
+// Built tells if the node (l, i) is built.
+func (t *Tree) Built(level, index int) (bool, error) {
+	_, built, err := t.place(level, index)
+	return built, err
 }
 
 // place returns the position of the node (l, i) if it is built.
@@ -327,12 +294,16 @@ func (t *Tree) place(level, index int) (Position, bool, error) {
 
 	records, err := t.level(level)
 	if err != nil {
-		return Position{}, false, err
+		return Position{}, false, fmt.Errorf("open node level index: %w", err)
 	}
 
 	record, found, err := records.Read(index)
 	if err != nil || !found {
-		return Position{}, false, err
+		if err != nil {
+			return Position{}, false, fmt.Errorf("read node level record: %w", err)
+		}
+
+		return Position{}, false, nil
 	}
 
 	place := decodePosition(record)
@@ -340,52 +311,83 @@ func (t *Tree) place(level, index int) (Position, bool, error) {
 	return place, place.length > 0, nil
 }
 
-// removeLevels removes the level indexes, which are not valid without the order
-// index.
-func (t *Tree) removeLevels() error {
-	paths, err := filepath.Glob(filepath.Join(t.dir, "tree-*.idx"))
+// Add writes a new node to the tree.
+func (t *Tree) Add(node Node) error {
+	if t.failure != nil {
+		return fmt.Errorf("tree needs a reopen after a failed write: %w", t.failure)
+	}
+
+	err := t.check(node)
 	if err != nil {
-		return fmt.Errorf("list level indexes: %w", err)
+		return fmt.Errorf("validate node: %w", err)
 	}
 
-	for _, path := range paths {
-		err = os.Remove(path)
-		if err != nil {
-			return fmt.Errorf("remove level index: %w", err)
-		}
+	line, err := json.Marshal(node)
+	if err != nil {
+		return fmt.Errorf("encode node: %w", err)
+	}
+
+	place, err := t.journal.Write(t.clock().Format(time.DateOnly), append(line, '\n'))
+	if err == nil {
+		err = t.addIndex(NodeKey{node.Level, node.Index}, place)
+	}
+
+	if err != nil {
+		t.failure = err
+		return fmt.Errorf("write node (%d, %d): %w", node.Level, node.Index, err)
 	}
 
 	return nil
 }
 
-// writeLevel writes the position of node to its level index.
-func (t *Tree) writeLevel(node NodeKey, place Position) error {
-	level, err := t.level(node.level)
+// Node reads the node (l, i) if it is built.
+func (t *Tree) Node(level, index int) (Node, bool, error) {
+	place, built, err := t.place(level, index)
+	if err != nil || !built {
+		if err != nil {
+			return Node{}, false, fmt.Errorf("find node position: %w", err)
+		}
+
+		return Node{}, false, nil
+	}
+
+	line, err := t.journal.Read(place)
 	if err != nil {
-		return err
+		return Node{}, false, fmt.Errorf("read node (%d, %d): %w", level, index, err)
 	}
 
-	record := make([]byte, positionBytes)
-	encodePosition(record, place)
+	var node Node
 
-	return level.Write(node.index, record)
-}
+	err = json.Unmarshal(line, &node)
+	if err != nil {
+		return Node{}, false, fmt.Errorf("decode node (%d, %d): %w", level, index, err)
+	}
 
-// level returns the index of level, and opens it at first use.
-func (t *Tree) level(level int) (*Records, error) {
-	index, open := t.levels[level]
-	if open {
-		return index, nil
+	if node.Level != level || node.Index != index {
+		return Node{}, false, fmt.Errorf("index of node (%d, %d) points to node (%d, %d)", level, index, node.Level, node.Index)
 	}
 
-	index, err := openRecords(filepath.Join(t.dir, fmt.Sprintf("tree-%d.idx", level)), positionBytes)
-	if err != nil {
-		return nil, err
+	return node, true, nil
+}
+
+// Keys returns the keys of the built nodes, ordered by level and then by index.
+func (t *Tree) Keys() ([]NodeKey, error) {
+	keys := make([]NodeKey, 0, t.order.count)
+	for n := range t.order.count {
+		record, _, err := t.order.Read(n)
+		if err != nil {
+			return nil, err
+		}
+
+		key, _ := treeDecodeRecord(record)
+		keys = append(keys, key)
 	}
 
-	t.levels[level] = index
+	slices.SortFunc(keys, func(a, b NodeKey) int {
+		return cmp.Or(cmp.Compare(a.level, b.level), cmp.Compare(a.index, b.index))
+	})
 
-	return index, nil
+	return keys, nil
 }
 
 // Count returns the number of built nodes.
@@ -393,23 +395,6 @@ func (t *Tree) Count() int {
 	return t.order.count
 }
 
-// Node is one line of the summary tree. Node (l, i) covers the 2^l messages
-// from i*2^l on. A node of level 0 holds one message. A node of a higher level
-// merges its two children, (l-1, 2i) and (l-1, 2i+1). Size is the UTF-8 byte
-// length of Text.
-type Node struct {
-	Level int    `json:"l"`
-	Index int    `json:"i"`
-	Text  string `json:"text"`
-	Size  int    `json:"size"`
-}
-
-// NodeKey names the node (l, i).
-type NodeKey struct {
-	level int
-	index int
-}
-
 // treeDecodeRecord reads the key and the position of a record of
 // index/tree.idx.
 func treeDecodeRecord(record []byte) (NodeKey, Position) {
@@ -420,3 +405,46 @@ func treeDecodeRecord(record []byte) (NodeKey, Position) {
 
 	return key, decodePosition(record)
 }
+
+// nodeEnd returns the ID after the last message of node (l, i).
+func nodeEnd(level, index int) int {
+	return (index + 1) << level
+}
+
+// freeLeaf returns a text-only leaf when its source fits in a node. Images need
+// a model description even when their text fits.
+func freeLeaf(message Message) (Node, bool) {
+	text := leafSource(message)
+	if len(text) > nodeBytes || len(imageFiles(message.Text)) > 0 {
+		return Node{}, false
+	}
+
+	return Node{
+		Level: 0,
+		Index: message.ID,
+		Text:  text,
+		Size:  len(text),
+	}, true
+}
+
+// leafSource returns the text that the leaf of message holds or summarizes.
+func leafSource(message Message) string {
+	return message.Kind + ": " + message.Text
+}
+
+// freeMerge returns the parent of two sibling nodes when both texts, joined by
+// a newline, fit in a node. Such a parent needs no model call.
+func freeMerge(left, right Node) (Node, bool) {
+	if left.Size+1+right.Size > nodeBytes {
+		return Node{}, false
+	}
+
+	text := left.Text + "\n" + right.Text
+
+	return Node{
+		Level: left.Level + 1,
+		Index: left.Index / 2,
+		Text:  text,
+		Size:  len(text),
+	}, true
+}