X-Git-Url: https://git.giacomolli.uk/onechat/blobdiff_plain/000ae97acb21a1c0c90e9278177a74411ba6eab2..fc0503be49438775c5d08499046852df59bd7377:/turn_test.go?ds=sidebyside diff --git a/turn_test.go b/turn_test.go index 2763965..3c6bb06 100644 --- a/turn_test.go +++ b/turn_test.go @@ -3,7 +3,6 @@ package main import ( "context" "encoding/json" - "log" "net/http" "path/filepath" "strings" @@ -19,7 +18,7 @@ func TestTurnRunsToolsUntilTheFinalAnswer(t *testing.T) { server.reply = func(w http.ResponseWriter, call ModelFakeCall) { if call.number == 0 { modelWriteEvents(w, modelCompletedEvent([]any{ - modelReasoningItem("rs_1", "opaque thought"), + turnReasoningItem("rs_1", "opaque thought"), modelFunctionCallItem("fc_1", "call_1", "shell", `{"command":"echo hi"}`), }, 100, 0, 5)) @@ -58,9 +57,11 @@ func TestTurnRunsToolsUntilTheFinalAnswer(t *testing.T) { "echo: hi\n", "reply: It said hi.", }, test.Log(t)) + require.Equal(t, []string{"user", "busy", "tool", "echo", "reply", "idle"}, test.events.Kinds()) var turns []string + for _, record := range modelReadUsage(t, test.usagePath) { turns = append(turns, record.Turn) } @@ -106,11 +107,13 @@ func TestMessageDuringATurnJoinsTheNextRequest(t *testing.T) { require.NoError(t, err) <-arrived + _, err = test.loop.Submit(terminalSource, "second, during the turn") require.NoError(t, err) close(release) test.events.WaitFor(t, "idle", 1) + input := server.Call(1).body["input"].([]any) require.Equal(t, map[string]any{ "role": "user", @@ -140,6 +143,7 @@ func TestMessageDuringTheFinalRequestStartsANewTurn(t *testing.T) { require.NoError(t, err) <-arrived + _, err = test.loop.Submit(terminalSource, "second") require.NoError(t, err) close(release) @@ -208,7 +212,7 @@ func TestInterruptEndsTheTurn(t *testing.T) { func TestFailedCallEndsTheTurnWithoutARetry(t *testing.T) { server := newModelFake(t) - server.reply = func(w http.ResponseWriter, call ModelFakeCall) { + server.reply = func(w http.ResponseWriter, _ ModelFakeCall) { http.Error(w, "busy", http.StatusServiceUnavailable) } @@ -221,7 +225,7 @@ func TestFailedCallEndsTheTurnWithoutARetry(t *testing.T) { require.Equal(t, 1, server.Calls()) require.Contains(t, test.events.Last("error"), "HTTP 503") require.Equal(t, 1, strings.Count(test.logs.String(), "\n")) - require.Contains(t, test.logs.String(), "turn failed: ") + require.Contains(t, test.logs.String(), "<3>turn failed: ") require.Contains(t, test.logs.String(), "HTTP 503") require.NotContains(t, test.logs.String(), "hello") } @@ -237,6 +241,7 @@ func TestViewMessagesOnlyGrowBetweenTurns(t *testing.T) { first := server.Call(1).body["input"].([]any) second := server.Call(2).body["input"].([]any) + require.Len(t, first, 4) require.Len(t, second, 5) require.Equal(t, first[:2], second[:2]) @@ -259,11 +264,65 @@ type TurnTestLoop struct { usagePath string } +// TurnEventRecorder keeps the events of a loop. +type TurnEventRecorder struct { + mutex sync.Mutex + events []Event +} + +func newTestTurnLoop(t *testing.T, server *ModelFake) *TurnTestLoop { + t.Helper() + + return newTestTurnLoopWith(t, server, newFakeSummarizer()) +} + +// newTestTurnLoopWith returns a running loop on a new agent whose builds use +// summarizer and whose commands run in a temporary directory. +func newTestTurnLoopWith(t *testing.T, server *ModelFake, summarizer Summarizer) *TurnTestLoop { + t.Helper() + + agent, err := openAgent(filepath.Join(t.TempDir(), "agent")) + require.NoError(t, err) + + agent.config.Workdir = t.TempDir() + clock := newTestClock() + memory, err := openMemory(t.Context(), t.TempDir(), clock.Now, summarizer, discardLog()) + require.NoError(t, err) + + test := &TurnTestLoop{ + agent: agent, + memory: memory, + events: &TurnEventRecorder{}, + logs: &SyncBuffer{}, + usagePath: filepath.Join(t.TempDir(), "usage.jsonl"), + } + + client := newModelClient(server.URL, ModelAPIKey(modelTestKey), agent.CacheKey(), test.usagePath) + whatsapp, _, _ := whatsappStartTest(t, t.TempDir(), clock) + test.loop, err = newTurnLoop(agent, memory, newTestSubagents(t), whatsapp, newTestSchedules(t, clock.Now), client, clock.Now, daemonTestLogger(test.logs), test.events.Add) + require.NoError(t, err) + + ctx, cancel := context.WithCancel(t.Context()) + done := make(chan error) + go func() { + done <- test.loop.Run(ctx) + }() + + t.Cleanup(func() { + cancel() + require.ErrorIs(t, <-done, context.Canceled) + require.NoError(t, memory.Close()) + }) + + return test +} + // Log returns the logged messages as "kind: text". func (l *TurnTestLoop) Log(t *testing.T) []string { t.Helper() var lines []string + for id := range l.memory.Len() { message, err := l.memory.Message(id) require.NoError(t, err) @@ -274,12 +333,6 @@ func (l *TurnTestLoop) Log(t *testing.T) []string { return lines } -// TurnEventRecorder keeps the events of a loop. -type TurnEventRecorder struct { - mutex sync.Mutex - events []Event -} - // WaitFor waits until count events of kind came. func (r *TurnEventRecorder) WaitFor(t *testing.T, kind string, count int) { t.Helper() @@ -317,6 +370,13 @@ func (r *TurnEventRecorder) Last(kind string) string { return r.LastEvent(kind).Text } +func (r *TurnEventRecorder) Add(event Event) { + r.mutex.Lock() + defer r.mutex.Unlock() + + r.events = append(r.events, event) +} + // LastEvent returns the last event of kind. func (r *TurnEventRecorder) LastEvent(kind string) Event { r.mutex.Lock() @@ -331,17 +391,13 @@ func (r *TurnEventRecorder) LastEvent(kind string) Event { return Event{} } -func (r *TurnEventRecorder) Add(event Event) { - r.mutex.Lock() - defer r.mutex.Unlock() - - r.events = append(r.events, event) -} - -func newTestTurnLoop(t *testing.T, server *ModelFake) *TurnTestLoop { - t.Helper() - - return newTestTurnLoopWith(t, server, newFakeSummarizer()) +func turnReasoningItem(id, encrypted string) map[string]any { + return map[string]any{ + "type": "reasoning", + "id": id, + "encrypted_content": encrypted, + "summary": []any{}, + } } // turnInputTexts returns the text of each user message of the input of call. @@ -360,12 +416,14 @@ func turnInputTexts(t *testing.T, call ModelFakeCall) []string { require.NoError(t, json.Unmarshal(call.raw, &body)) var texts []string + for _, item := range body.Input { if item.Role != "user" { continue } var text strings.Builder + for _, part := range item.Content { text.WriteString(part.Text) } @@ -375,43 +433,3 @@ func turnInputTexts(t *testing.T, call ModelFakeCall) []string { return texts } - -// newTestTurnLoopWith returns a running loop on a new agent whose builds use -// summarizer and whose commands run in a temporary directory. -func newTestTurnLoopWith(t *testing.T, server *ModelFake, summarizer Summarizer) *TurnTestLoop { - t.Helper() - - agent, err := openAgent(t.TempDir()) - require.NoError(t, err) - - agent.config.Workdir = t.TempDir() - clock := newTestClock() - memory, err := openMemory(t.Context(), t.TempDir(), clock.Now, summarizer, discardLog()) - require.NoError(t, err) - - test := &TurnTestLoop{ - agent: agent, - memory: memory, - events: &TurnEventRecorder{}, - logs: &SyncBuffer{}, - usagePath: filepath.Join(t.TempDir(), "usage.jsonl"), - } - - client := newModelClient(server.URL, ModelAPIKey(modelTestKey), agent.CacheKey(), test.usagePath) - whatsapp, _, _ := whatsappStartTest(t, t.TempDir(), clock) - test.loop, err = newTurnLoop(agent, memory, newTestSubagents(t), whatsapp, newTestSchedules(t, clock.Now), client, clock.Now, log.New(test.logs, "", 0), test.events.Add) - require.NoError(t, err) - ctx, cancel := context.WithCancel(t.Context()) - done := make(chan error) - go func() { - done <- test.loop.Run(ctx) - }() - - t.Cleanup(func() { - cancel() - require.ErrorIs(t, <-done, context.Canceled) - require.NoError(t, memory.Close()) - }) - - return test -}