X-Git-Url: https://git.giacomolli.uk/onechat/blobdiff_plain/4a931bccab5879ab708647b2df8dadc052f128bf..000ae97acb21a1c0c90e9278177a74411ba6eab2:/memory.go?ds=sidebyside

diff --git a/memory.go b/memory.go
index 0fd7d83..bcc1095 100644
--- a/memory.go
+++ b/memory.go
@@ -5,6 +5,7 @@ import (
 	"encoding/json"
 	"errors"
 	"fmt"
+	"log"
 	"os"
 	"path/filepath"
 	"strings"
@@ -22,9 +23,9 @@ var (
 )
 
 // openMemory opens the chat in root and starts the builds that are ready. The
-// clock gives the dates of new messages and the days of new nodes. The builds
-// stop when ctx ends or at Close.
-func openMemory(ctx context.Context, root string, clock func() time.Time, summarizer Summarizer) (*Memory, error) {
+// clock gives the dates of new messages and the days of new nodes. logger gets
+// one line for each failed build. The builds stop when ctx ends or at Close.
+func openMemory(ctx context.Context, root string, clock func() time.Time, summarizer Summarizer, logger *log.Logger) (*Memory, error) {
 	log, err := openLog(ctx, root, clock)
 	if err != nil {
 		return nil, err
@@ -35,6 +36,7 @@ func openMemory(ctx context.Context, root string, clock func() time.Time, summar
 		log:        log,
 		queue:      newBuildQueue(),
 		summarizer: summarizer,
+		logger:     logger,
 		changed:    make(chan struct{}),
 	}
 
@@ -106,6 +108,7 @@ type Memory struct {
 	cview      *View
 	queue      *BuildQueue
 	summarizer Summarizer
+	logger     *log.Logger
 	ctx        context.Context
 	cancel     context.CancelFunc
 	workers    sync.WaitGroup
@@ -136,24 +139,30 @@ func (m *Memory) Close() error {
 // Append logs a new message, adds it to the tree and the views, and starts the
 // builds that are ready, also the failed ones.
 func (m *Memory) Append(kind, text string) (Message, error) {
+	return m.AppendDated(kind, text, "")
+}
+
+// AppendDated appends a message with its RFC 3339 date and builds its nodes. An
+// empty date uses the clock, as Append does.
+func (m *Memory) AppendDated(kind, text, date string) (Message, error) {
 	m.mutex.Lock()
 	defer m.mutex.Unlock()
 
-	message, err := m.log.Append(kind, text)
+	message, err := m.log.AppendDated(kind, text, date)
 	if err != nil {
-		return Message{}, err
+		return Message{}, fmt.Errorf("append dated log message: %w", err)
 	}
 
 	defer m.syncer.Written()
 
 	err = m.addMessage(message)
 	if err != nil {
-		return Message{}, err
+		return Message{}, fmt.Errorf("add dated message to memory: %w", err)
 	}
 
 	err = m.save()
 	if err != nil {
-		return Message{}, err
+		return Message{}, fmt.Errorf("save dated message view: %w", err)
 	}
 
 	m.queue.Retry()
@@ -394,7 +403,9 @@ func (m *Memory) dispatch() {
 
 		task, err := m.task(key)
 		if err != nil {
+			m.failed(key, fmt.Errorf("read its input: %w", err))
 			m.queue.Fail(key)
+
 			continue
 		}
 
@@ -561,6 +572,10 @@ func (m *Memory) run(key NodeKey, task Task) {
 		m.syncer.Written()
 	}
 
+	if err != nil && m.ctx.Err() == nil {
+		m.failed(key, err)
+	}
+
 	if err != nil {
 		m.queue.Fail(key)
 	} else {
@@ -572,6 +587,11 @@ func (m *Memory) run(key NodeKey, task Task) {
 	m.dispatch()
 }
 
+// failed logs the failed build of the node key with its cause err.
+func (m *Memory) failed(key NodeKey, err error) {
+	m.logger.Printf("summary failed, retry at next message: node (%d, %d): %v", key.level, key.index, err)
+}
+
 // ready tells if both children of the node key are built and the node is not.
 func (m *Memory) ready(key NodeKey) (bool, error) {
 	for _, index := range []int{2 * key.index, 2*key.index + 1} {
@@ -613,6 +633,79 @@ func (m *Memory) Context(level, index int) string {
 	return m.context(level, index)
 }
 
+// rebuildComplete checks the whole tree, not only the visible view. A compactor
+// can fail without an HTTP error, for example with an empty line. Such a failed
+// merge can be absent from the view and must prevent a swap.
+func (m *Memory) rebuildComplete() error {
+	m.mutex.Lock()
+	defer m.mutex.Unlock()
+
+	if len(m.queue.leaves) != 0 || len(m.queue.queued) != 0 {
+		return errors.New("rebuild failed: summary tree has unfinished nodes")
+	}
+
+	return nil
+}
+
+// buildModelLines returns the sizes of the nodes that the model wrote, in tree
+// order. A node that fits its source with no model call is not one of them.
+func (m *Memory) buildModelLines() ([]int, error) {
+	m.mutex.Lock()
+	defer m.mutex.Unlock()
+
+	keys, err := m.tree.Keys()
+	if err != nil {
+		return nil, fmt.Errorf("list built nodes: %w", err)
+	}
+
+	var sizes []int
+	for _, key := range keys {
+		var node Node
+
+		node, _, err = m.tree.Node(key.level, key.index)
+		if err != nil {
+			return nil, fmt.Errorf("read built node: %w", err)
+		}
+
+		var model bool
+
+		model, err = m.buildBuiltByModel(key)
+		if err != nil {
+			return nil, fmt.Errorf("check model source: %w", err)
+		}
+
+		if model {
+			sizes = append(sizes, node.Size)
+		}
+	}
+
+	return sizes, nil
+}
+
+// buildBuiltByModel tells if the node key needed a model call: a leaf whose
+// source is over nodeBytes, or a merge whose children do not fit in one node.
+func (m *Memory) buildBuiltByModel(key NodeKey) (bool, error) {
+	if key.level == 0 {
+		message, err := m.log.Message(key.index)
+		if err != nil {
+			return false, fmt.Errorf("read leaf source: %w", err)
+		}
+
+		_, free := freeLeaf(message)
+
+		return !free, nil
+	}
+
+	left, right, err := m.children(key)
+	if err != nil {
+		return false, fmt.Errorf("read merge sources: %w", err)
+	}
+
+	_, free := freeMerge(left, right)
+
+	return !free, nil
+}
+
 // Task is the build of the node (Level, Index). Context is the compaction view
 // for the build. Input holds the source "kind: text" of a leaf, or the two
 // child lines of a merge with their newlines as spaces.