git.giacomolli.uk - onechat/blob - turn_test.go

   1 package main
   2
   3 import (
   4         "context"
   5         "encoding/json"
   6         "errors"
   7         "net/http"
   8         "path/filepath"
   9         "strings"
  10         "sync"
  11         "testing"
  12         "time"
  13
  14         "github.com/stretchr/testify/require"
  15 )
  16
  17 func TestTurnRunsToolsUntilTheFinalAnswer(t *testing.T) {
  18         server := newFakeResponses(t)
  19         server.reply = func(w http.ResponseWriter, call fakeCall) {
  20                 if call.number == 0 {
  21                         writeEvents(w, completedEvent([]any{
  22                                 reasoningItem("rs_1", "opaque thought"),
  23                                 functionCallItem("fc_1", "call_1", "shell", `{"command":"echo hi"}`),
  24                         }, 100, 0, 5))
  25
  26                         return
  27                 }
  28
  29                 writeEvents(w, completedEvent([]any{messageItem("It said hi.")}, 120, 100, 5))
  30         }
  31
  32         test := newTestLoop(t, server)
  33         _, err := test.loop.Submit("Run echo hi.")
  34         require.NoError(t, err)
  35         test.events.WaitFor(t, "idle", 1)
  36         require.Equal(t, 2, server.Calls())
  37
  38         first := server.Call(0).body
  39         second := server.Call(1).body
  40         instructions, err := test.agent.Prompt()
  41         require.NoError(t, err)
  42         require.Equal(t, instructions, first["instructions"])
  43         require.Equal(t, test.agent.config.Model, first["model"])
  44
  45         // The second request begins with the whole first one.
  46         firstInput := first["input"].([]any)
  47         secondInput := second["input"].([]any)
  48         require.Equal(t, firstInput, secondInput[:len(firstInput)])
  49         require.Equal(t, []any{
  50                 map[string]any{"type": "reasoning", "encrypted_content": "opaque thought", "summary": []any{}},
  51                 map[string]any{"type": "function_call", "call_id": "call_1", "name": "shell", "arguments": `{"command":"echo hi"}`},
  52                 map[string]any{"type": "function_call_output", "call_id": "call_1", "output": "hi\n"},
  53         }, secondInput[len(firstInput):])
  54
  55         require.Equal(t, []string{
  56                 "user: Run echo hi.",
  57                 `tool: shell {"command":"echo hi"}`,
  58                 "echo: hi\n",
  59                 "reply: It said hi.",
  60         }, test.Log(t))
  61         require.Equal(t, []string{"user", "busy", "tool", "echo", "reply", "idle"}, test.events.Kinds())
  62 }
  63
  64 func TestFirstRequestHoldsTheViewTheStateAndTheMessage(t *testing.T) {
  65         server := newFakeResponses(t)
  66         test := newTestLoop(t, server)
  67         _, err := test.memory.Append("user", "an earlier message")
  68         require.NoError(t, err)
  69
  70         _, err = test.loop.Submit("The new message.")
  71         require.NoError(t, err)
  72         test.events.WaitFor(t, "idle", 1)
  73
  74         require.Equal(t, []string{
  75                 "<chat>\n0+1|user: an earlier message\n",
  76                 "</chat>\n\nNow: Thursday 2026-01-01 12:00 +00:00",
  77                 "The new message.",
  78         }, inputTexts(t, server.Call(0)))
  79 }
  80
  81 func TestMessageDuringATurnJoinsTheNextRequest(t *testing.T) {
  82         server := newFakeResponses(t)
  83         arrived := make(chan struct{})
  84         release := make(chan struct{})
  85         server.reply = func(w http.ResponseWriter, call fakeCall) {
  86                 if call.number == 0 {
  87                         close(arrived)
  88                         <-release
  89                         writeEvents(w, completedEvent([]any{functionCallItem("fc_1", "call_1", "date", `{"id":0}`)}, 100, 0, 5))
  90
  91                         return
  92                 }
  93
  94                 writeEvents(w, completedEvent([]any{messageItem("done")}, 100, 0, 5))
  95         }
  96
  97         test := newTestLoop(t, server)
  98         _, err := test.loop.Submit("first")
  99         require.NoError(t, err)
 100
 101         <-arrived
 102         _, err = test.loop.Submit("second, during the turn")
 103         require.NoError(t, err)
 104         close(release)
 105
 106         test.events.WaitFor(t, "idle", 1)
 107         input := server.Call(1).body["input"].([]any)
 108         require.Equal(t, map[string]any{
 109                 "role":    "user",
 110                 "content": []any{map[string]any{"type": "input_text", "text": "second, during the turn"}},
 111         }, input[len(input)-1])
 112
 113         time.Sleep(50 * time.Millisecond)
 114         require.Equal(t, 2, server.Calls())
 115         require.Equal(t, 1, test.events.Count("busy"))
 116 }
 117
 118 func TestMessageDuringTheFinalRequestStartsANewTurn(t *testing.T) {
 119         server := newFakeResponses(t)
 120         arrived := make(chan struct{})
 121         release := make(chan struct{})
 122         server.reply = func(w http.ResponseWriter, call fakeCall) {
 123                 if call.number == 0 {
 124                         close(arrived)
 125                         <-release
 126                 }
 127
 128                 writeEvents(w, completedEvent([]any{messageItem("answer")}, 100, 0, 5))
 129         }
 130
 131         test := newTestLoop(t, server)
 132         _, err := test.loop.Submit("first")
 133         require.NoError(t, err)
 134
 135         <-arrived
 136         _, err = test.loop.Submit("second")
 137         require.NoError(t, err)
 138         close(release)
 139
 140         test.events.WaitFor(t, "idle", 2)
 141         require.Equal(t, 2, server.Calls())
 142
 143         texts := inputTexts(t, server.Call(1))
 144         require.Equal(t, "second", texts[len(texts)-1])
 145         require.Contains(t, strings.Join(texts, ""), "|reply: answer\n")
 146 }
 147
 148 func TestTurnWaitsForTheSummaries(t *testing.T) {
 149         server := newFakeResponses(t)
 150         summarizer := newFakeSummarizer()
 151         summarizer.gate = make(chan struct{})
 152         test := newTestLoopWith(t, server, summarizer, false)
 153
 154         _, err := test.memory.Append("echo", strings.Repeat("long output ", 100))
 155         require.NoError(t, err)
 156
 157         _, err = test.loop.Submit("go on")
 158         require.NoError(t, err)
 159
 160         time.Sleep(50 * time.Millisecond)
 161         require.Equal(t, 0, server.Calls())
 162
 163         close(summarizer.gate)
 164         test.events.WaitFor(t, "idle", 1)
 165         require.Equal(t, 1, server.Calls())
 166         require.NotContains(t, inputTexts(t, server.Call(0))[0], pending)
 167 }
 168
 169 func TestInterruptEndsTheTurn(t *testing.T) {
 170         server := newFakeResponses(t)
 171         arrived := make(chan struct{})
 172         release := make(chan struct{})
 173         server.reply = func(w http.ResponseWriter, call fakeCall) {
 174                 if call.number == 0 {
 175                         close(arrived)
 176                         <-release
 177                 }
 178
 179                 writeEvents(w, completedEvent([]any{messageItem("answer")}, 100, 0, 5))
 180         }
 181
 182         test := newTestLoop(t, server)
 183         _, err := test.loop.Submit("a long task")
 184         require.NoError(t, err)
 185
 186         <-arrived
 187         require.True(t, test.loop.Interrupt())
 188         test.events.WaitFor(t, "idle", 1)
 189         close(release)
 190
 191         require.Equal(t, "interrupted", test.events.Last("error"))
 192         require.Equal(t, []string{"user: a long task"}, test.Log(t))
 193         require.False(t, test.loop.Interrupt())
 194
 195         _, err = test.loop.Submit("again")
 196         require.NoError(t, err)
 197         test.events.WaitFor(t, "idle", 2)
 198         require.Equal(t, "reply: answer", test.Log(t)[2])
 199 }
 200
 201 func TestFailedCallEndsTheTurnWithoutARetry(t *testing.T) {
 202         server := newFakeResponses(t)
 203         server.reply = func(w http.ResponseWriter, call fakeCall) {
 204                 http.Error(w, "busy", http.StatusServiceUnavailable)
 205         }
 206
 207         test := newTestLoop(t, server)
 208         _, err := test.loop.Submit("hello")
 209         require.NoError(t, err)
 210         test.events.WaitFor(t, "idle", 1)
 211
 212         time.Sleep(50 * time.Millisecond)
 213         require.Equal(t, 1, server.Calls())
 214         require.Contains(t, test.events.Last("error"), "HTTP 503")
 215 }
 216
 217 func TestViewMessagesOnlyGrowBetweenTurns(t *testing.T) {
 218         server := newFakeResponses(t)
 219         test := newTestLoop(t, server)
 220         for n, text := range []string{"one", "two", "three"} {
 221                 _, err := test.loop.Submit(text)
 222                 require.NoError(t, err)
 223                 test.events.WaitFor(t, "idle", n+1)
 224         }
 225
 226         first := server.Call(1).body["input"].([]any)
 227         second := server.Call(2).body["input"].([]any)
 228         require.Len(t, first, 4)
 229         require.Len(t, second, 5)
 230         require.Equal(t, first[:2], second[:2])
 231         require.Equal(t, []string{
 232                 "<chat>\n",
 233                 "0+1|user: one\n1+1|reply: a line\n",
 234                 "2+1|user: two\n3+1|reply: a line\n",
 235                 "</chat>\n\nNow: Thursday 2026-01-01 12:00 +00:00",
 236                 "three",
 237         }, inputTexts(t, server.Call(2)))
 238 }
 239
 240 func TestWarmUpEndsWhereTheNextTurnReadsTheCache(t *testing.T) {
 241         server := newFakeResponses(t)
 242         server.reply = func(w http.ResponseWriter, call fakeCall) {
 243                 if call.number == 1 {
 244                         writeEvents(w, map[string]any{"type": "response.output_item.added", "item": reasoningItem("rs_1", "")})
 245                         return
 246                 }
 247
 248                 writeEvents(w, completedEvent([]any{messageItem("answer")}, 100, 0, 5))
 249         }
 250
 251         test := newTestLoopWith(t, server, newFakeSummarizer(), true)
 252         _, err := test.loop.Submit("one")
 253         require.NoError(t, err)
 254         require.Eventually(t, func() bool {
 255                 return server.Calls() == 2
 256         }, 10*time.Second, time.Millisecond)
 257
 258         _, err = test.loop.Submit("two")
 259         require.NoError(t, err)
 260         test.events.WaitFor(t, "idle", 2)
 261         require.Eventually(t, func() bool {
 262                 return server.Calls() == 4
 263         }, 10*time.Second, time.Millisecond)
 264
 265         warmUp := server.Call(1).body
 266         require.Equal(t, server.Call(0).body["instructions"], warmUp["instructions"])
 267         require.Equal(t, server.Call(0).body["tools"], warmUp["tools"])
 268         require.Equal(t, []string{"<chat>\n", "0+1|user: one\n1+1|reply: answer\n"}, inputTexts(t, server.Call(1)))
 269
 270         // The next turn begins with the whole warm-up, so its message end after
 271         // the view is the end of the warm-up entry.
 272         warmInput := warmUp["input"].([]any)
 273         turnInput := server.Call(2).body["input"].([]any)
 274         require.Equal(t, warmInput, turnInput[:len(warmInput)])
 275         require.Equal(t, "</chat>\n\nNow: Thursday 2026-01-01 12:00 +00:00", inputTexts(t, server.Call(2))[len(warmInput)])
 276
 277         turns := []string{}
 278         for _, record := range readUsage(t, test.usagePath) {
 279                 turns = append(turns, record.Turn)
 280         }
 281
 282         require.Equal(t, []string{"first", "warmup", "first", "warmup"}, turns)
 283         require.Equal(t, []string{"user: one", "reply: answer", "user: two", "reply: answer"}, test.Log(t))
 284 }
 285
 286 // testLoop is a loop that runs in the background of a test, with its parts.
 287 type testLoop struct {
 288         agent     *Agent
 289         memory    *Memory
 290         loop      *loop
 291         events    *eventRecorder
 292         usagePath string
 293 }
 294
 295 // Log returns the logged messages as "kind: text".
 296 func (l *testLoop) Log(t *testing.T) []string {
 297         t.Helper()
 298
 299         var lines []string
 300         for id := range l.memory.Len() {
 301                 message, err := l.memory.Message(id)
 302                 require.NoError(t, err)
 303
 304                 lines = append(lines, message.Kind+": "+message.Text)
 305         }
 306
 307         return lines
 308 }
 309
 310 // eventRecorder keeps the events of a loop.
 311 type eventRecorder struct {
 312         mutex  sync.Mutex
 313         events []Event
 314 }
 315
 316 func (r *eventRecorder) Add(event Event) {
 317         r.mutex.Lock()
 318         defer r.mutex.Unlock()
 319
 320         r.events = append(r.events, event)
 321 }
 322
 323 func (r *eventRecorder) Kinds() []string {
 324         r.mutex.Lock()
 325         defer r.mutex.Unlock()
 326
 327         kinds := make([]string, len(r.events))
 328         for n, event := range r.events {
 329                 kinds[n] = event.Kind
 330         }
 331
 332         return kinds
 333 }
 334
 335 func (r *eventRecorder) Count(kind string) int {
 336         count := 0
 337         for _, each := range r.Kinds() {
 338                 if each == kind {
 339                         count++
 340                 }
 341         }
 342
 343         return count
 344 }
 345
 346 // Last returns the text of the last event of kind.
 347 func (r *eventRecorder) Last(kind string) string {
 348         r.mutex.Lock()
 349         defer r.mutex.Unlock()
 350
 351         for n := len(r.events) - 1; n >= 0; n-- {
 352                 if r.events[n].Kind == kind {
 353                         return r.events[n].Text
 354                 }
 355         }
 356
 357         return ""
 358 }
 359
 360 // WaitFor waits until count events of kind came.
 361 func (r *eventRecorder) WaitFor(t *testing.T, kind string, count int) {
 362         t.Helper()
 363
 364         require.Eventually(t, func() bool {
 365                 return r.Count(kind) >= count
 366         }, 10*time.Second, time.Millisecond)
 367 }
 368
 369 func newTestLoop(t *testing.T, server *fakeResponses) *testLoop {
 370         t.Helper()
 371
 372         return newTestLoopWith(t, server, newFakeSummarizer(), false)
 373 }
 374
 375 // newTestLoopWith returns a running loop on a new agent whose builds use
 376 // summarizer and whose commands run in a temporary directory. warmUp turns the
 377 // warm-up on.
 378 func newTestLoopWith(t *testing.T, server *fakeResponses, summarizer Summarizer, warmUp bool) *testLoop {
 379         t.Helper()
 380
 381         agent, err := openAgent(t.TempDir())
 382         require.NoError(t, err)
 383
 384         agent.config.Workdir = t.TempDir()
 385         clock := newTestClock()
 386         memory, err := openMemory(t.Context(), t.TempDir(), clock.Now, summarizer)
 387         require.NoError(t, err)
 388
 389         test := &testLoop{
 390                 agent:     agent,
 391                 memory:    memory,
 392                 events:    &eventRecorder{},
 393                 usagePath: filepath.Join(t.TempDir(), "usage.jsonl"),
 394         }
 395
 396         client := newClient(server.URL, apiKey(testKey), agent.CacheKey(), test.usagePath)
 397         test.loop, err = newLoop(agent, memory, client, clock.Now, test.events.Add)
 398         require.NoError(t, err)
 399
 400         test.loop.warmUp = warmUp
 401         ctx, cancel := context.WithCancel(t.Context())
 402         done := make(chan error)
 403         go func() {
 404                 done <- test.loop.Run(ctx)
 405         }()
 406
 407         t.Cleanup(func() {
 408                 cancel()
 409                 require.True(t, errors.Is(<-done, context.Canceled))
 410                 require.NoError(t, memory.Close())
 411         })
 412
 413         return test
 414 }
 415
 416 // inputTexts returns the text of each user message of the input of call.
 417 func inputTexts(t *testing.T, call fakeCall) []string {
 418         t.Helper()
 419
 420         var body struct {
 421                 Input []struct {
 422                         Role    string `json:"role"`
 423                         Content []struct {
 424                                 Text string `json:"text"`
 425                         } `json:"content"`
 426                 } `json:"input"`
 427         }
 428
 429         require.NoError(t, json.Unmarshal(call.raw, &body))
 430
 431         var texts []string
 432         for _, item := range body.Input {
 433                 if item.Role != "user" {
 434                         continue
 435                 }
 436
 437                 var text strings.Builder
 438                 for _, part := range item.Content {
 439                         text.WriteString(part.Text)
 440                 }
 441
 442                 texts = append(texts, text.String())
 443         }
 444
 445         return texts
 446 }
 447
 448 func reasoningItem(id, encrypted string) map[string]any {
 449         return map[string]any{
 450                 "type":              "reasoning",
 451                 "id":                id,
 452                 "encrypted_content": encrypted,
 453                 "summary":           []any{},
 454         }
 455 }
 456
 457 func functionCallItem(id, callID, name, arguments string) map[string]any {
 458         return map[string]any{
 459                 "type":      "function_call",
 460                 "id":        id,
 461                 "call_id":   callID,
 462                 "name":      name,
 463                 "arguments": arguments,
 464         }
 465 }