git.giacomolli.uk - onechat/blob - turn_test.go

   1 package main
   2
   3 import (
   4         "context"
   5         "encoding/json"
   6         "net/http"
   7         "path/filepath"
   8         "strings"
   9         "sync"
  10         "testing"
  11         "time"
  12
  13         "github.com/stretchr/testify/require"
  14 )
  15
  16 func TestTurnRunsToolsUntilTheFinalAnswer(t *testing.T) {
  17         server := newModelFake(t)
  18         server.reply = func(w http.ResponseWriter, call ModelFakeCall) {
  19                 if call.number == 0 {
  20                         modelWriteEvents(w, modelCompletedEvent([]any{
  21                                 modelReasoningItem("rs_1", "opaque thought"),
  22                                 modelFunctionCallItem("fc_1", "call_1", "shell", `{"command":"echo hi"}`),
  23                         }, 100, 0, 5))
  24
  25                         return
  26                 }
  27
  28                 modelWriteEvents(w, modelCompletedEvent([]any{modelMessageItem("It said hi.")}, 120, 100, 5))
  29         }
  30
  31         test := newTestTurnLoop(t, server)
  32         _, err := test.loop.Submit(terminalSource, "Run echo hi.")
  33         require.NoError(t, err)
  34         test.events.WaitFor(t, "idle", 1)
  35         require.Equal(t, 2, server.Calls())
  36
  37         first := server.Call(0).body
  38         second := server.Call(1).body
  39         instructions, err := test.agent.Prompt()
  40         require.NoError(t, err)
  41         require.Equal(t, instructions, first["instructions"])
  42         require.Equal(t, test.agent.config.Model, first["model"])
  43
  44         // The second request begins with the whole first one.
  45         firstInput := first["input"].([]any)
  46         secondInput := second["input"].([]any)
  47         require.Equal(t, firstInput, secondInput[:len(firstInput)])
  48         require.Equal(t, []any{
  49                 map[string]any{"type": "reasoning", "encrypted_content": "opaque thought", "summary": []any{}},
  50                 map[string]any{"type": "function_call", "call_id": "call_1", "name": "shell", "arguments": `{"command":"echo hi"}`},
  51                 map[string]any{"type": "function_call_output", "call_id": "call_1", "output": "hi\n"},
  52         }, secondInput[len(firstInput):])
  53
  54         require.Equal(t, []string{
  55                 "user: Run echo hi.",
  56                 `tool: shell {"command":"echo hi"}`,
  57                 "echo: hi\n",
  58                 "reply: It said hi.",
  59         }, test.Log(t))
  60
  61         require.Equal(t, []string{"user", "busy", "tool", "echo", "reply", "idle"}, test.events.Kinds())
  62
  63         var turns []string
  64
  65         for _, record := range modelReadUsage(t, test.usagePath) {
  66                 turns = append(turns, record.Turn)
  67         }
  68
  69         require.Equal(t, []string{"first", "next"}, turns)
  70 }
  71
  72 func TestFirstRequestHoldsTheViewTheStateAndTheMessage(t *testing.T) {
  73         server := newModelFake(t)
  74         test := newTestTurnLoop(t, server)
  75         _, err := test.memory.Append("user", "an earlier message")
  76         require.NoError(t, err)
  77
  78         _, err = test.loop.Submit(terminalSource, "The new message.")
  79         require.NoError(t, err)
  80         test.events.WaitFor(t, "idle", 1)
  81
  82         require.Equal(t, []string{
  83                 "<chat>\n0+1|user: an earlier message\n",
  84                 "</chat>\n\nNow: Thursday 2026-01-01 12:00 +00:00",
  85                 "The new message.",
  86         }, turnInputTexts(t, server.Call(0)))
  87 }
  88
  89 func TestMessageDuringATurnJoinsTheNextRequest(t *testing.T) {
  90         server := newModelFake(t)
  91         arrived := make(chan struct{})
  92         release := make(chan struct{})
  93         server.reply = func(w http.ResponseWriter, call ModelFakeCall) {
  94                 if call.number == 0 {
  95                         close(arrived)
  96                         <-release
  97                         modelWriteEvents(w, modelCompletedEvent([]any{modelFunctionCallItem("fc_1", "call_1", "date", `{"id":0}`)}, 100, 0, 5))
  98
  99                         return
 100                 }
 101
 102                 modelWriteEvents(w, modelCompletedEvent([]any{modelMessageItem("done")}, 100, 0, 5))
 103         }
 104
 105         test := newTestTurnLoop(t, server)
 106         _, err := test.loop.Submit(terminalSource, "first")
 107         require.NoError(t, err)
 108
 109         <-arrived
 110
 111         _, err = test.loop.Submit(terminalSource, "second, during the turn")
 112         require.NoError(t, err)
 113         close(release)
 114
 115         test.events.WaitFor(t, "idle", 1)
 116
 117         input := server.Call(1).body["input"].([]any)
 118         require.Equal(t, map[string]any{
 119                 "role":    "user",
 120                 "content": []any{map[string]any{"type": "input_text", "text": "second, during the turn"}},
 121         }, input[len(input)-1])
 122
 123         time.Sleep(50 * time.Millisecond)
 124         require.Equal(t, 2, server.Calls())
 125         require.Equal(t, 1, test.events.Count("busy"))
 126 }
 127
 128 func TestMessageDuringTheFinalRequestStartsANewTurn(t *testing.T) {
 129         server := newModelFake(t)
 130         arrived := make(chan struct{})
 131         release := make(chan struct{})
 132         server.reply = func(w http.ResponseWriter, call ModelFakeCall) {
 133                 if call.number == 0 {
 134                         close(arrived)
 135                         <-release
 136                 }
 137
 138                 modelWriteEvents(w, modelCompletedEvent([]any{modelMessageItem("answer")}, 100, 0, 5))
 139         }
 140
 141         test := newTestTurnLoop(t, server)
 142         _, err := test.loop.Submit(terminalSource, "first")
 143         require.NoError(t, err)
 144
 145         <-arrived
 146
 147         _, err = test.loop.Submit(terminalSource, "second")
 148         require.NoError(t, err)
 149         close(release)
 150
 151         test.events.WaitFor(t, "idle", 2)
 152         require.Equal(t, 2, server.Calls())
 153
 154         texts := turnInputTexts(t, server.Call(1))
 155         require.Equal(t, "second", texts[len(texts)-1])
 156         require.Contains(t, strings.Join(texts, ""), "|reply: answer\n")
 157 }
 158
 159 func TestTurnWaitsForTheSummaries(t *testing.T) {
 160         server := newModelFake(t)
 161         summarizer := newFakeSummarizer()
 162         summarizer.gate = make(chan struct{})
 163         test := newTestTurnLoopWith(t, server, summarizer)
 164
 165         _, err := test.memory.Append("echo", strings.Repeat("long output ", 100))
 166         require.NoError(t, err)
 167
 168         _, err = test.loop.Submit(terminalSource, "go on")
 169         require.NoError(t, err)
 170
 171         time.Sleep(50 * time.Millisecond)
 172         require.Equal(t, 0, server.Calls())
 173
 174         close(summarizer.gate)
 175         test.events.WaitFor(t, "idle", 1)
 176         require.Equal(t, 1, server.Calls())
 177         require.NotContains(t, turnInputTexts(t, server.Call(0))[0], viewPending)
 178 }
 179
 180 func TestInterruptEndsTheTurn(t *testing.T) {
 181         server := newModelFake(t)
 182         arrived := make(chan struct{})
 183         release := make(chan struct{})
 184         server.reply = func(w http.ResponseWriter, call ModelFakeCall) {
 185                 if call.number == 0 {
 186                         close(arrived)
 187                         <-release
 188                 }
 189
 190                 modelWriteEvents(w, modelCompletedEvent([]any{modelMessageItem("answer")}, 100, 0, 5))
 191         }
 192
 193         test := newTestTurnLoop(t, server)
 194         _, err := test.loop.Submit(terminalSource, "a long task")
 195         require.NoError(t, err)
 196
 197         <-arrived
 198         require.True(t, test.loop.Interrupt())
 199         test.events.WaitFor(t, "idle", 1)
 200         close(release)
 201
 202         require.Equal(t, "interrupted", test.events.Last("error"))
 203         require.Equal(t, []string{"user: a long task"}, test.Log(t))
 204         require.False(t, test.loop.Interrupt())
 205
 206         _, err = test.loop.Submit(terminalSource, "again")
 207         require.NoError(t, err)
 208         test.events.WaitFor(t, "idle", 2)
 209         require.Equal(t, "reply: answer", test.Log(t)[2])
 210         require.Empty(t, test.logs.String())
 211 }
 212
 213 func TestFailedCallEndsTheTurnWithoutARetry(t *testing.T) {
 214         server := newModelFake(t)
 215         server.reply = func(w http.ResponseWriter, _ ModelFakeCall) {
 216                 http.Error(w, "busy", http.StatusServiceUnavailable)
 217         }
 218
 219         test := newTestTurnLoop(t, server)
 220         _, err := test.loop.Submit(terminalSource, "hello")
 221         require.NoError(t, err)
 222         test.events.WaitFor(t, "idle", 1)
 223
 224         time.Sleep(50 * time.Millisecond)
 225         require.Equal(t, 1, server.Calls())
 226         require.Contains(t, test.events.Last("error"), "HTTP 503")
 227         require.Equal(t, 1, strings.Count(test.logs.String(), "\n"))
 228         require.Contains(t, test.logs.String(), "<3>turn failed: ")
 229         require.Contains(t, test.logs.String(), "HTTP 503")
 230         require.NotContains(t, test.logs.String(), "hello")
 231 }
 232
 233 func TestViewMessagesOnlyGrowBetweenTurns(t *testing.T) {
 234         server := newModelFake(t)
 235         test := newTestTurnLoop(t, server)
 236         for n, text := range []string{"one", "two", "three"} {
 237                 _, err := test.loop.Submit(terminalSource, text)
 238                 require.NoError(t, err)
 239                 test.events.WaitFor(t, "idle", n+1)
 240         }
 241
 242         first := server.Call(1).body["input"].([]any)
 243         second := server.Call(2).body["input"].([]any)
 244
 245         require.Len(t, first, 4)
 246         require.Len(t, second, 5)
 247         require.Equal(t, first[:2], second[:2])
 248         require.Equal(t, []string{
 249                 "<chat>\n",
 250                 "0+1|user: one\n1+1|reply: a line\n",
 251                 "2+1|user: two\n3+1|reply: a line\n",
 252                 "</chat>\n\nNow: Thursday 2026-01-01 12:00 +00:00",
 253                 "three",
 254         }, turnInputTexts(t, server.Call(2)))
 255 }
 256
 257 // TurnTestLoop is a loop that runs in the background of a test, with its parts.
 258 type TurnTestLoop struct {
 259         agent     *Agent
 260         memory    *Memory
 261         loop      *TurnLoop
 262         events    *TurnEventRecorder
 263         logs      *SyncBuffer
 264         usagePath string
 265 }
 266
 267 // TurnEventRecorder keeps the events of a loop.
 268 type TurnEventRecorder struct {
 269         mutex  sync.Mutex
 270         events []Event
 271 }
 272
 273 func newTestTurnLoop(t *testing.T, server *ModelFake) *TurnTestLoop {
 274         t.Helper()
 275
 276         return newTestTurnLoopWith(t, server, newFakeSummarizer())
 277 }
 278
 279 // newTestTurnLoopWith returns a running loop on a new agent whose builds use
 280 // summarizer and whose commands run in a temporary directory.
 281 func newTestTurnLoopWith(t *testing.T, server *ModelFake, summarizer Summarizer) *TurnTestLoop {
 282         t.Helper()
 283
 284         agent, err := openAgent(filepath.Join(t.TempDir(), "agent"))
 285         require.NoError(t, err)
 286
 287         agent.config.Workdir = t.TempDir()
 288         clock := newTestClock()
 289         memory, err := openMemory(t.Context(), t.TempDir(), clock.Now, summarizer, discardLog())
 290         require.NoError(t, err)
 291
 292         test := &TurnTestLoop{
 293                 agent:     agent,
 294                 memory:    memory,
 295                 events:    &TurnEventRecorder{},
 296                 logs:      &SyncBuffer{},
 297                 usagePath: filepath.Join(t.TempDir(), "usage.jsonl"),
 298         }
 299
 300         client := newModelClient(server.URL, ModelAPIKey(modelTestKey), agent.CacheKey(), test.usagePath)
 301         whatsapp, _, _ := whatsappStartTest(t, t.TempDir(), clock)
 302         test.loop, err = newTurnLoop(agent, memory, newTestSubagents(t), whatsapp, newTestSchedules(t, clock.Now), client, clock.Now, daemonTestLogger(test.logs), test.events.Add)
 303         require.NoError(t, err)
 304
 305         ctx, cancel := context.WithCancel(t.Context())
 306         done := make(chan error)
 307         go func() {
 308                 done <- test.loop.Run(ctx)
 309         }()
 310
 311         t.Cleanup(func() {
 312                 cancel()
 313                 require.ErrorIs(t, <-done, context.Canceled)
 314                 require.NoError(t, memory.Close())
 315         })
 316
 317         return test
 318 }
 319
 320 // Log returns the logged messages as "kind: text".
 321 func (l *TurnTestLoop) Log(t *testing.T) []string {
 322         t.Helper()
 323
 324         var lines []string
 325
 326         for id := range l.memory.Len() {
 327                 message, err := l.memory.Message(id)
 328                 require.NoError(t, err)
 329
 330                 lines = append(lines, message.Kind+": "+message.Text)
 331         }
 332
 333         return lines
 334 }
 335
 336 // WaitFor waits until count events of kind came.
 337 func (r *TurnEventRecorder) WaitFor(t *testing.T, kind string, count int) {
 338         t.Helper()
 339
 340         require.Eventually(t, func() bool {
 341                 return r.Count(kind) >= count
 342         }, 10*time.Second, time.Millisecond)
 343 }
 344
 345 func (r *TurnEventRecorder) Kinds() []string {
 346         r.mutex.Lock()
 347         defer r.mutex.Unlock()
 348
 349         kinds := make([]string, len(r.events))
 350         for n, event := range r.events {
 351                 kinds[n] = event.Kind
 352         }
 353
 354         return kinds
 355 }
 356
 357 func (r *TurnEventRecorder) Count(kind string) int {
 358         count := 0
 359         for _, each := range r.Kinds() {
 360                 if each == kind {
 361                         count++
 362                 }
 363         }
 364
 365         return count
 366 }
 367
 368 // Last returns the text of the last event of kind.
 369 func (r *TurnEventRecorder) Last(kind string) string {
 370         return r.LastEvent(kind).Text
 371 }
 372
 373 // LastEvent returns the last event of kind.
 374 func (r *TurnEventRecorder) LastEvent(kind string) Event {
 375         r.mutex.Lock()
 376         defer r.mutex.Unlock()
 377
 378         for n := len(r.events) - 1; n >= 0; n-- {
 379                 if r.events[n].Kind == kind {
 380                         return r.events[n]
 381                 }
 382         }
 383
 384         return Event{}
 385 }
 386
 387 func (r *TurnEventRecorder) Add(event Event) {
 388         r.mutex.Lock()
 389         defer r.mutex.Unlock()
 390
 391         r.events = append(r.events, event)
 392 }
 393
 394 // turnInputTexts returns the text of each user message of the input of call.
 395 func turnInputTexts(t *testing.T, call ModelFakeCall) []string {
 396         t.Helper()
 397
 398         var body struct {
 399                 Input []struct {
 400                         Role    string `json:"role"`
 401                         Content []struct {
 402                                 Text string `json:"text"`
 403                         } `json:"content"`
 404                 } `json:"input"`
 405         }
 406
 407         require.NoError(t, json.Unmarshal(call.raw, &body))
 408
 409         var texts []string
 410
 411         for _, item := range body.Input {
 412                 if item.Role != "user" {
 413                         continue
 414                 }
 415
 416                 var text strings.Builder
 417
 418                 for _, part := range item.Content {
 419                         text.WriteString(part.Text)
 420                 }
 421
 422                 texts = append(texts, text.String())
 423         }
 424
 425         return texts
 426 }
 427
 428 func modelReasoningItem(id, encrypted string) map[string]any {
 429         return map[string]any{
 430                 "type":              "reasoning",
 431                 "id":                id,
 432                 "encrypted_content": encrypted,
 433                 "summary":           []any{},
 434         }
 435 }