From: Anderson A. Giacomolli Date: Fri, 6 Nov 2026 15:06:45 +0000 (-0300) Subject: Replay a synthetic log through the core X-Git-Url: http://git.giacomolli.uk/onechat/commitdiff_plain/442c68e1d384a7da0d9ab039704bf763a1c3ac35 Replay a synthetic log through the core The exit check of the memory core runs a synthetic log of 100,000 messages through the core with the fake summarizer. The log mixes short user lines, replies, notes, reports, tool calls, and tool output, and one output in a hundred is over 30 KB. The core has no reader of the optchat log, because that belongs to the history import. After each message the test checks that the chat view is at most 128,000 bytes and the compaction view at most 32,000, and that a batch merges the chat view to 64,000 bytes or less. It checks each rendered line: a built line holds its node, a long message has a summary and not its text, and an unbuilt line holds only the pending mark. A restart in the middle and at the end loads the same views and view.json byte for byte. Issue: beb9a70 --- diff --git a/replay_test.go b/replay_test.go new file mode 100644 index 0000000..4151d7e --- /dev/null +++ b/replay_test.go @@ -0,0 +1,185 @@ +package main + +import ( + "fmt" + "math/rand/v2" + "os" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/stretchr/testify/require" +) + +// syntheticMessages is the length of the synthetic log. +const syntheticMessages = 100000 + +// TestReplaySyntheticLog runs a synthetic log through the memory core with the +// fake summarizer. After each message it waits for the builds, as a turn does. +// It checks that each view stays within its sawtooth, that no view line holds a +// whole long message or a part of an unbuilt one, and that a restart in the +// middle and at the end loads the same views byte for byte. +func TestReplaySyntheticLog(t *testing.T) { + start := time.Now() + root := t.TempDir() + clock := newTestClock() + random := rand.New(rand.NewPCG(1, 2)) + memory, err := openMemory(t.Context(), root, clock.Now, newFakeSummarizer()) + require.NoError(t, err) + + defer func() { + require.NoError(t, memory.Close()) + }() + + batches := 0 + low := chatHigh + for id := range syntheticMessages { + if id == syntheticMessages/2 { + memory = requireRestart(t, memory, root, clock) + } + + clock.now = clock.now.Add(time.Minute) + kind, text := syntheticMessage(random, id) + lines := viewLines(memory) + _, err = memory.Append(kind, text) + require.NoError(t, err) + + size, merged := requireViews(t, memory, lines) + if merged { + batches++ + low = min(low, size) + } + + require.NoError(t, memory.Wait(t.Context())) + } + + memory = requireRestart(t, memory, root, clock) + require.Equal(t, syntheticMessages, memory.log.Len()) + require.Greater(t, batches, 0) + + require.NoError(t, memory.Wait(t.Context())) + t.Logf("%d messages, %d nodes, %d batches in %s", syntheticMessages, len(memory.tree.nodes), batches, time.Since(start).Round(time.Second)) + t.Logf("chat view: %d lines, %d bytes, lowest after a batch %d bytes", len(memory.chat.lines), memory.chat.Size(memory.tree), low) + t.Logf("compaction view: %d lines, %d bytes", len(memory.cview.lines), memory.cview.Size(memory.tree)) +} + +// syntheticMessage returns the kind and the text of message id. Most messages +// are tool calls and their output. Users write short lines. One output in a +// hundred is over 30 KB. +func syntheticMessage(random *rand.Rand, id int) (string, string) { + roll := random.IntN(100) + switch { + case roll < 5: + return "user", syntheticText(id, 10+random.IntN(200)) + case roll < 10: + return "reply", syntheticText(id, 100+random.IntN(1500)) + case roll < 12: + return "note", syntheticText(id, 50+random.IntN(600)) + case roll < 13: + return "work", syntheticText(id, 300+random.IntN(3000)) + case roll < 56: + return "tool", syntheticText(id, 60+random.IntN(400)) + case roll < 57: + return "echo", syntheticText(id, 30000+random.IntN(30000)) + default: + return "echo", syntheticText(id, 20+random.IntN(4000)) + } +} + +// syntheticText returns a text of about size bytes that starts with its message +// ID and holds some non-ASCII characters. +func syntheticText(id, size int) string { + head := fmt.Sprintf("m%d ", id) + words := "ação data line café " + + return head + strings.Repeat(words, max(1, (size-len(head))/len(words))) +} + +// requireViews checks the views after a new message, while no build changes the +// tree. Each view is at most its high limit, and a batch merges the chat view +// to its low limit. It returns the size of the chat view and if a batch merged +// it, from the number of lines before the message. +func requireViews(t *testing.T, memory *Memory, lines int) (int, bool) { + t.Helper() + + memory.mutex.Lock() + defer memory.mutex.Unlock() + + size := memory.chat.Size(memory.tree) + require.LessOrEqual(t, size, chatHigh) + require.LessOrEqual(t, memory.cview.Size(memory.tree), compactionHigh) + + merged := len(memory.chat.lines) < lines + if merged { + require.LessOrEqual(t, size, chatLow) + } + + requireOnlySummaries(t, memory) + + return size, merged +} + +// requireOnlySummaries checks each line of the chat view as the model reads it. +// A built line holds its node, a long message has a summary and not its text, +// and an unbuilt line holds only the pending mark. The caller holds the lock of +// memory. +func requireOnlySummaries(t *testing.T, memory *Memory) { + t.Helper() + + rendered := strings.Split(memory.chat.Render(memory.tree), "\n") + require.Len(t, rendered, len(memory.chat.lines)+2) + + for n, line := range memory.chat.lines { + head := fmt.Sprintf("%d+%d|", line.index< nodeBytes { + require.NotEqual(t, source, node.Text, "message %d", line.index) + } + } + } +} + +// viewLines returns the number of lines in the chat view of memory. +func viewLines(memory *Memory) int { + memory.mutex.Lock() + defer memory.mutex.Unlock() + + return len(memory.chat.lines) +} + +// requireRestart closes memory, opens it again, and checks that the views and +// view.json stay the same byte for byte. +func requireRestart(t *testing.T, memory *Memory, root string, clock *testClock) *Memory { + t.Helper() + + require.NoError(t, memory.Wait(t.Context())) + + view := memory.View() + context := memory.Context(0, memory.log.Len()) + saved, err := os.ReadFile(filepath.Join(root, "view.json")) + require.NoError(t, err) + require.NoError(t, memory.Close()) + + memory, err = openMemory(t.Context(), root, clock.Now, newFakeSummarizer()) + require.NoError(t, err) + require.NoError(t, memory.Wait(t.Context())) + require.Equal(t, view, memory.View()) + require.Equal(t, context, memory.Context(0, memory.log.Len())) + + loaded, err := os.ReadFile(filepath.Join(root, "view.json")) + require.NoError(t, err) + require.Equal(t, saved, loaded) + + return memory +}