X-Git-Url: https://git.giacomolli.uk/onechat/blobdiff_plain/02cdaff8be27f37f44db8892bb12554f2d702323..a3b9849a54a6cd8a4cc5b1ae03a39aab31322f7b:/turn_test.go diff --git a/turn_test.go b/turn_test.go index c378469..5d93e08 100644 --- a/turn_test.go +++ b/turn_test.go @@ -3,7 +3,7 @@ package main import ( "context" "encoding/json" - "errors" + "log" "net/http" "path/filepath" "strings" @@ -29,7 +29,7 @@ func TestTurnRunsToolsUntilTheFinalAnswer(t *testing.T) { modelWriteEvents(w, modelCompletedEvent([]any{modelMessageItem("It said hi.")}, 120, 100, 5)) } - test := newTestLoop(t, server) + test := newTestTurnLoop(t, server) _, err := test.loop.Submit(terminalSource, "Run echo hi.") require.NoError(t, err) test.events.WaitFor(t, "idle", 1) @@ -70,7 +70,7 @@ func TestTurnRunsToolsUntilTheFinalAnswer(t *testing.T) { func TestFirstRequestHoldsTheViewTheStateAndTheMessage(t *testing.T) { server := newModelFake(t) - test := newTestLoop(t, server) + test := newTestTurnLoop(t, server) _, err := test.memory.Append("user", "an earlier message") require.NoError(t, err) @@ -82,7 +82,7 @@ func TestFirstRequestHoldsTheViewTheStateAndTheMessage(t *testing.T) { "\n0+1|user: an earlier message\n", "\n\nNow: Thursday 2026-01-01 12:00 +00:00", "The new message.", - }, inputTexts(t, server.Call(0))) + }, turnInputTexts(t, server.Call(0))) } func TestMessageDuringATurnJoinsTheNextRequest(t *testing.T) { @@ -101,7 +101,7 @@ func TestMessageDuringATurnJoinsTheNextRequest(t *testing.T) { modelWriteEvents(w, modelCompletedEvent([]any{modelMessageItem("done")}, 100, 0, 5)) } - test := newTestLoop(t, server) + test := newTestTurnLoop(t, server) _, err := test.loop.Submit(terminalSource, "first") require.NoError(t, err) @@ -135,7 +135,7 @@ func TestMessageDuringTheFinalRequestStartsANewTurn(t *testing.T) { modelWriteEvents(w, modelCompletedEvent([]any{modelMessageItem("answer")}, 100, 0, 5)) } - test := newTestLoop(t, server) + test := newTestTurnLoop(t, server) _, err := test.loop.Submit(terminalSource, "first") require.NoError(t, err) @@ -147,7 +147,7 @@ func TestMessageDuringTheFinalRequestStartsANewTurn(t *testing.T) { test.events.WaitFor(t, "idle", 2) require.Equal(t, 2, server.Calls()) - texts := inputTexts(t, server.Call(1)) + texts := turnInputTexts(t, server.Call(1)) require.Equal(t, "second", texts[len(texts)-1]) require.Contains(t, strings.Join(texts, ""), "|reply: answer\n") } @@ -156,7 +156,7 @@ func TestTurnWaitsForTheSummaries(t *testing.T) { server := newModelFake(t) summarizer := newFakeSummarizer() summarizer.gate = make(chan struct{}) - test := newTestLoopWith(t, server, summarizer) + test := newTestTurnLoopWith(t, server, summarizer) _, err := test.memory.Append("echo", strings.Repeat("long output ", 100)) require.NoError(t, err) @@ -170,7 +170,7 @@ func TestTurnWaitsForTheSummaries(t *testing.T) { close(summarizer.gate) test.events.WaitFor(t, "idle", 1) require.Equal(t, 1, server.Calls()) - require.NotContains(t, inputTexts(t, server.Call(0))[0], pending) + require.NotContains(t, turnInputTexts(t, server.Call(0))[0], viewPending) } func TestInterruptEndsTheTurn(t *testing.T) { @@ -186,7 +186,7 @@ func TestInterruptEndsTheTurn(t *testing.T) { modelWriteEvents(w, modelCompletedEvent([]any{modelMessageItem("answer")}, 100, 0, 5)) } - test := newTestLoop(t, server) + test := newTestTurnLoop(t, server) _, err := test.loop.Submit(terminalSource, "a long task") require.NoError(t, err) @@ -203,15 +203,16 @@ func TestInterruptEndsTheTurn(t *testing.T) { require.NoError(t, err) test.events.WaitFor(t, "idle", 2) require.Equal(t, "reply: answer", test.Log(t)[2]) + require.Empty(t, test.logs.String()) } func TestFailedCallEndsTheTurnWithoutARetry(t *testing.T) { server := newModelFake(t) - server.reply = func(w http.ResponseWriter, call ModelFakeCall) { + server.reply = func(w http.ResponseWriter, _ ModelFakeCall) { http.Error(w, "busy", http.StatusServiceUnavailable) } - test := newTestLoop(t, server) + test := newTestTurnLoop(t, server) _, err := test.loop.Submit(terminalSource, "hello") require.NoError(t, err) test.events.WaitFor(t, "idle", 1) @@ -219,11 +220,15 @@ func TestFailedCallEndsTheTurnWithoutARetry(t *testing.T) { time.Sleep(50 * time.Millisecond) require.Equal(t, 1, server.Calls()) require.Contains(t, test.events.Last("error"), "HTTP 503") + require.Equal(t, 1, strings.Count(test.logs.String(), "\n")) + require.Contains(t, test.logs.String(), "turn failed: ") + require.Contains(t, test.logs.String(), "HTTP 503") + require.NotContains(t, test.logs.String(), "hello") } func TestViewMessagesOnlyGrowBetweenTurns(t *testing.T) { server := newModelFake(t) - test := newTestLoop(t, server) + test := newTestTurnLoop(t, server) for n, text := range []string{"one", "two", "three"} { _, err := test.loop.Submit(terminalSource, text) require.NoError(t, err) @@ -241,20 +246,21 @@ func TestViewMessagesOnlyGrowBetweenTurns(t *testing.T) { "2+1|user: two\n3+1|reply: a line\n", "\n\nNow: Thursday 2026-01-01 12:00 +00:00", "three", - }, inputTexts(t, server.Call(2))) + }, turnInputTexts(t, server.Call(2))) } -// testLoop is a loop that runs in the background of a test, with its parts. -type testLoop struct { +// TurnTestLoop is a loop that runs in the background of a test, with its parts. +type TurnTestLoop struct { agent *Agent memory *Memory - loop *loop - events *eventRecorder + loop *TurnLoop + events *TurnEventRecorder + logs *SyncBuffer usagePath string } // Log returns the logged messages as "kind: text". -func (l *testLoop) Log(t *testing.T) []string { +func (l *TurnTestLoop) Log(t *testing.T) []string { t.Helper() var lines []string @@ -268,20 +274,22 @@ func (l *testLoop) Log(t *testing.T) []string { return lines } -// eventRecorder keeps the events of a loop. -type eventRecorder struct { +// TurnEventRecorder keeps the events of a loop. +type TurnEventRecorder struct { mutex sync.Mutex events []Event } -func (r *eventRecorder) Add(event Event) { - r.mutex.Lock() - defer r.mutex.Unlock() +// WaitFor waits until count events of kind came. +func (r *TurnEventRecorder) WaitFor(t *testing.T, kind string, count int) { + t.Helper() - r.events = append(r.events, event) + require.Eventually(t, func() bool { + return r.Count(kind) >= count + }, 10*time.Second, time.Millisecond) } -func (r *eventRecorder) Kinds() []string { +func (r *TurnEventRecorder) Kinds() []string { r.mutex.Lock() defer r.mutex.Unlock() @@ -293,7 +301,7 @@ func (r *eventRecorder) Kinds() []string { return kinds } -func (r *eventRecorder) Count(kind string) int { +func (r *TurnEventRecorder) Count(kind string) int { count := 0 for _, each := range r.Kinds() { if each == kind { @@ -305,37 +313,72 @@ func (r *eventRecorder) Count(kind string) int { } // Last returns the text of the last event of kind. -func (r *eventRecorder) Last(kind string) string { +func (r *TurnEventRecorder) Last(kind string) string { + return r.LastEvent(kind).Text +} + +// LastEvent returns the last event of kind. +func (r *TurnEventRecorder) LastEvent(kind string) Event { r.mutex.Lock() defer r.mutex.Unlock() for n := len(r.events) - 1; n >= 0; n-- { if r.events[n].Kind == kind { - return r.events[n].Text + return r.events[n] } } - return "" + return Event{} } -// WaitFor waits until count events of kind came. -func (r *eventRecorder) WaitFor(t *testing.T, kind string, count int) { +func (r *TurnEventRecorder) Add(event Event) { + r.mutex.Lock() + defer r.mutex.Unlock() + + r.events = append(r.events, event) +} + +func newTestTurnLoop(t *testing.T, server *ModelFake) *TurnTestLoop { t.Helper() - require.Eventually(t, func() bool { - return r.Count(kind) >= count - }, 10*time.Second, time.Millisecond) + return newTestTurnLoopWith(t, server, newFakeSummarizer()) } -func newTestLoop(t *testing.T, server *ModelFake) *testLoop { +// turnInputTexts returns the text of each user message of the input of call. +func turnInputTexts(t *testing.T, call ModelFakeCall) []string { t.Helper() - return newTestLoopWith(t, server, newFakeSummarizer()) + var body struct { + Input []struct { + Role string `json:"role"` + Content []struct { + Text string `json:"text"` + } `json:"content"` + } `json:"input"` + } + + require.NoError(t, json.Unmarshal(call.raw, &body)) + + var texts []string + for _, item := range body.Input { + if item.Role != "user" { + continue + } + + var text strings.Builder + for _, part := range item.Content { + text.WriteString(part.Text) + } + + texts = append(texts, text.String()) + } + + return texts } -// newTestLoopWith returns a running loop on a new agent whose builds use +// newTestTurnLoopWith returns a running loop on a new agent whose builds use // summarizer and whose commands run in a temporary directory. -func newTestLoopWith(t *testing.T, server *ModelFake, summarizer Summarizer) *testLoop { +func newTestTurnLoopWith(t *testing.T, server *ModelFake, summarizer Summarizer) *TurnTestLoop { t.Helper() agent, err := openAgent(t.TempDir()) @@ -343,18 +386,20 @@ func newTestLoopWith(t *testing.T, server *ModelFake, summarizer Summarizer) *te agent.config.Workdir = t.TempDir() clock := newTestClock() - memory, err := openMemory(t.Context(), t.TempDir(), clock.Now, summarizer) + memory, err := openMemory(t.Context(), t.TempDir(), clock.Now, summarizer, discardLog()) require.NoError(t, err) - test := &testLoop{ + test := &TurnTestLoop{ agent: agent, memory: memory, - events: &eventRecorder{}, + events: &TurnEventRecorder{}, + logs: &SyncBuffer{}, usagePath: filepath.Join(t.TempDir(), "usage.jsonl"), } client := newModelClient(server.URL, ModelAPIKey(modelTestKey), agent.CacheKey(), test.usagePath) - test.loop, err = newLoop(agent, memory, client, clock.Now, test.events.Add) + whatsapp, _, _ := whatsappStartTest(t, t.TempDir(), clock) + test.loop, err = newTurnLoop(agent, memory, newTestSubagents(t), whatsapp, newTestSchedules(t, clock.Now), client, clock.Now, log.New(test.logs, "", 0), test.events.Add) require.NoError(t, err) ctx, cancel := context.WithCancel(t.Context()) done := make(chan error) @@ -364,41 +409,9 @@ func newTestLoopWith(t *testing.T, server *ModelFake, summarizer Summarizer) *te t.Cleanup(func() { cancel() - require.True(t, errors.Is(<-done, context.Canceled)) + require.ErrorIs(t, <-done, context.Canceled) require.NoError(t, memory.Close()) }) return test } - -// inputTexts returns the text of each user message of the input of call. -func inputTexts(t *testing.T, call ModelFakeCall) []string { - t.Helper() - - var body struct { - Input []struct { - Role string `json:"role"` - Content []struct { - Text string `json:"text"` - } `json:"content"` - } `json:"input"` - } - - require.NoError(t, json.Unmarshal(call.raw, &body)) - - var texts []string - for _, item := range body.Input { - if item.Role != "user" { - continue - } - - var text strings.Builder - for _, part := range item.Content { - text.WriteString(part.Text) - } - - texts = append(texts, text.String()) - } - - return texts -}