git.giacomolli.uk - onechat/blob - turn_test.go

   1 package main
   2
   3 import (
   4         "context"
   5         "errors"
   6         "net/http"
   7         "strings"
   8         "sync"
   9         "testing"
  10         "time"
  11
  12         "github.com/stretchr/testify/require"
  13 )
  14
  15 func TestTurnRunsToolsUntilTheFinalAnswer(t *testing.T) {
  16         server := newFakeResponses(t)
  17         server.reply = func(w http.ResponseWriter, call fakeCall) {
  18                 if call.number == 0 {
  19                         writeEvents(w, completedEvent([]any{
  20                                 reasoningItem("rs_1", "opaque thought"),
  21                                 functionCallItem("fc_1", "call_1", "shell", `{"command":"echo hi"}`),
  22                         }, 100, 0, 5))
  23
  24                         return
  25                 }
  26
  27                 writeEvents(w, completedEvent([]any{messageItem("It said hi.")}, 120, 100, 5))
  28         }
  29
  30         test := newTestLoop(t, server)
  31         _, err := test.loop.Submit("Run echo hi.")
  32         require.NoError(t, err)
  33         test.events.WaitFor(t, "idle", 1)
  34         require.Equal(t, 2, server.Calls())
  35
  36         first := server.Call(0).body
  37         second := server.Call(1).body
  38         instructions, err := test.agent.Prompt()
  39         require.NoError(t, err)
  40         require.Equal(t, instructions, first["instructions"])
  41         require.Equal(t, test.agent.config.Model, first["model"])
  42
  43         // The second request begins with the whole first one.
  44         firstInput := first["input"].([]any)
  45         secondInput := second["input"].([]any)
  46         require.Equal(t, firstInput, secondInput[:len(firstInput)])
  47         require.Equal(t, []any{
  48                 map[string]any{"type": "reasoning", "encrypted_content": "opaque thought", "summary": []any{}},
  49                 map[string]any{"type": "function_call", "call_id": "call_1", "name": "shell", "arguments": `{"command":"echo hi"}`},
  50                 map[string]any{"type": "function_call_output", "call_id": "call_1", "output": "hi\n"},
  51         }, secondInput[len(firstInput):])
  52
  53         require.Equal(t, []string{
  54                 "user: Run echo hi.",
  55                 `tool: shell {"command":"echo hi"}`,
  56                 "echo: hi\n",
  57                 "reply: It said hi.",
  58         }, test.Log(t))
  59         require.Equal(t, []string{"user", "busy", "tool", "echo", "reply", "idle"}, test.events.Kinds())
  60 }
  61
  62 func TestFirstRequestHoldsTheViewTheStateAndTheMessage(t *testing.T) {
  63         server := newFakeResponses(t)
  64         test := newTestLoop(t, server)
  65         _, err := test.memory.Append("user", "an earlier message")
  66         require.NoError(t, err)
  67
  68         _, err = test.loop.Submit("The new message.")
  69         require.NoError(t, err)
  70         test.events.WaitFor(t, "idle", 1)
  71
  72         require.Equal(t, []string{
  73                 "<chat>\n0+1|user: an earlier message\n</chat>\n",
  74                 "Now: Thursday 2026-01-01 12:00 +00:00\n",
  75                 "The new message.",
  76         }, inputParts(t, server.Call(0)))
  77 }
  78
  79 func TestMessageDuringATurnJoinsTheNextRequest(t *testing.T) {
  80         server := newFakeResponses(t)
  81         arrived := make(chan struct{})
  82         release := make(chan struct{})
  83         server.reply = func(w http.ResponseWriter, call fakeCall) {
  84                 if call.number == 0 {
  85                         close(arrived)
  86                         <-release
  87                         writeEvents(w, completedEvent([]any{functionCallItem("fc_1", "call_1", "date", `{"id":0}`)}, 100, 0, 5))
  88
  89                         return
  90                 }
  91
  92                 writeEvents(w, completedEvent([]any{messageItem("done")}, 100, 0, 5))
  93         }
  94
  95         test := newTestLoop(t, server)
  96         _, err := test.loop.Submit("first")
  97         require.NoError(t, err)
  98
  99         <-arrived
 100         _, err = test.loop.Submit("second, during the turn")
 101         require.NoError(t, err)
 102         close(release)
 103
 104         test.events.WaitFor(t, "idle", 1)
 105         input := server.Call(1).body["input"].([]any)
 106         require.Equal(t, map[string]any{
 107                 "role":    "user",
 108                 "content": []any{map[string]any{"type": "input_text", "text": "second, during the turn"}},
 109         }, input[len(input)-1])
 110
 111         time.Sleep(50 * time.Millisecond)
 112         require.Equal(t, 2, server.Calls())
 113         require.Equal(t, 1, test.events.Count("busy"))
 114 }
 115
 116 func TestMessageDuringTheFinalRequestStartsANewTurn(t *testing.T) {
 117         server := newFakeResponses(t)
 118         arrived := make(chan struct{})
 119         release := make(chan struct{})
 120         server.reply = func(w http.ResponseWriter, call fakeCall) {
 121                 if call.number == 0 {
 122                         close(arrived)
 123                         <-release
 124                 }
 125
 126                 writeEvents(w, completedEvent([]any{messageItem("answer")}, 100, 0, 5))
 127         }
 128
 129         test := newTestLoop(t, server)
 130         _, err := test.loop.Submit("first")
 131         require.NoError(t, err)
 132
 133         <-arrived
 134         _, err = test.loop.Submit("second")
 135         require.NoError(t, err)
 136         close(release)
 137
 138         test.events.WaitFor(t, "idle", 2)
 139         require.Equal(t, 2, server.Calls())
 140
 141         parts := inputParts(t, server.Call(1))
 142         require.Equal(t, "second", parts[len(parts)-1])
 143         require.Contains(t, parts[0], "|reply: answer\n")
 144 }
 145
 146 func TestTurnWaitsForTheSummaries(t *testing.T) {
 147         server := newFakeResponses(t)
 148         summarizer := newFakeSummarizer()
 149         summarizer.gate = make(chan struct{})
 150         test := newTestLoopWith(t, server, summarizer)
 151
 152         _, err := test.memory.Append("echo", strings.Repeat("long output ", 100))
 153         require.NoError(t, err)
 154
 155         _, err = test.loop.Submit("go on")
 156         require.NoError(t, err)
 157
 158         time.Sleep(50 * time.Millisecond)
 159         require.Equal(t, 0, server.Calls())
 160
 161         close(summarizer.gate)
 162         test.events.WaitFor(t, "idle", 1)
 163         require.Equal(t, 1, server.Calls())
 164         require.NotContains(t, inputParts(t, server.Call(0))[0], pending)
 165 }
 166
 167 func TestInterruptEndsTheTurn(t *testing.T) {
 168         server := newFakeResponses(t)
 169         arrived := make(chan struct{})
 170         release := make(chan struct{})
 171         server.reply = func(w http.ResponseWriter, call fakeCall) {
 172                 if call.number == 0 {
 173                         close(arrived)
 174                         <-release
 175                 }
 176
 177                 writeEvents(w, completedEvent([]any{messageItem("answer")}, 100, 0, 5))
 178         }
 179
 180         test := newTestLoop(t, server)
 181         _, err := test.loop.Submit("a long task")
 182         require.NoError(t, err)
 183
 184         <-arrived
 185         require.True(t, test.loop.Interrupt())
 186         test.events.WaitFor(t, "idle", 1)
 187         close(release)
 188
 189         require.Equal(t, "interrupted", test.events.Last("error"))
 190         require.Equal(t, []string{"user: a long task"}, test.Log(t))
 191         require.False(t, test.loop.Interrupt())
 192
 193         _, err = test.loop.Submit("again")
 194         require.NoError(t, err)
 195         test.events.WaitFor(t, "idle", 2)
 196         require.Equal(t, "reply: answer", test.Log(t)[2])
 197 }
 198
 199 func TestFailedCallEndsTheTurnWithoutARetry(t *testing.T) {
 200         server := newFakeResponses(t)
 201         server.reply = func(w http.ResponseWriter, call fakeCall) {
 202                 http.Error(w, "busy", http.StatusServiceUnavailable)
 203         }
 204
 205         test := newTestLoop(t, server)
 206         _, err := test.loop.Submit("hello")
 207         require.NoError(t, err)
 208         test.events.WaitFor(t, "idle", 1)
 209
 210         time.Sleep(50 * time.Millisecond)
 211         require.Equal(t, 1, server.Calls())
 212         require.Contains(t, test.events.Last("error"), "HTTP 503")
 213 }
 214
 215 // testLoop is a loop that runs in the background of a test, with its parts.
 216 type testLoop struct {
 217         agent  *Agent
 218         memory *Memory
 219         loop   *loop
 220         events *eventRecorder
 221 }
 222
 223 // Log returns the logged messages as "kind: text".
 224 func (l *testLoop) Log(t *testing.T) []string {
 225         t.Helper()
 226
 227         var lines []string
 228         for id := range l.memory.Len() {
 229                 message, err := l.memory.Message(id)
 230                 require.NoError(t, err)
 231
 232                 lines = append(lines, message.Kind+": "+message.Text)
 233         }
 234
 235         return lines
 236 }
 237
 238 // eventRecorder keeps the events of a loop.
 239 type eventRecorder struct {
 240         mutex  sync.Mutex
 241         events []Event
 242 }
 243
 244 func (r *eventRecorder) Add(event Event) {
 245         r.mutex.Lock()
 246         defer r.mutex.Unlock()
 247
 248         r.events = append(r.events, event)
 249 }
 250
 251 func (r *eventRecorder) Kinds() []string {
 252         r.mutex.Lock()
 253         defer r.mutex.Unlock()
 254
 255         kinds := make([]string, len(r.events))
 256         for n, event := range r.events {
 257                 kinds[n] = event.Kind
 258         }
 259
 260         return kinds
 261 }
 262
 263 func (r *eventRecorder) Count(kind string) int {
 264         count := 0
 265         for _, each := range r.Kinds() {
 266                 if each == kind {
 267                         count++
 268                 }
 269         }
 270
 271         return count
 272 }
 273
 274 // Last returns the text of the last event of kind.
 275 func (r *eventRecorder) Last(kind string) string {
 276         r.mutex.Lock()
 277         defer r.mutex.Unlock()
 278
 279         for n := len(r.events) - 1; n >= 0; n-- {
 280                 if r.events[n].Kind == kind {
 281                         return r.events[n].Text
 282                 }
 283         }
 284
 285         return ""
 286 }
 287
 288 // WaitFor waits until count events of kind came.
 289 func (r *eventRecorder) WaitFor(t *testing.T, kind string, count int) {
 290         t.Helper()
 291
 292         require.Eventually(t, func() bool {
 293                 return r.Count(kind) >= count
 294         }, 10*time.Second, time.Millisecond)
 295 }
 296
 297 func newTestLoop(t *testing.T, server *fakeResponses) *testLoop {
 298         t.Helper()
 299
 300         return newTestLoopWith(t, server, newFakeSummarizer())
 301 }
 302
 303 // newTestLoopWith returns a running loop on a new agent whose builds use
 304 // summarizer and whose commands run in a temporary directory.
 305 func newTestLoopWith(t *testing.T, server *fakeResponses, summarizer Summarizer) *testLoop {
 306         t.Helper()
 307
 308         agent, err := openAgent(t.TempDir())
 309         require.NoError(t, err)
 310
 311         agent.config.Workdir = t.TempDir()
 312         clock := newTestClock()
 313         memory, err := openMemory(t.Context(), t.TempDir(), clock.Now, summarizer)
 314         require.NoError(t, err)
 315
 316         test := &testLoop{
 317                 agent:  agent,
 318                 memory: memory,
 319                 events: &eventRecorder{},
 320         }
 321
 322         test.loop = newLoop(agent, memory, newTestClient(t, server), clock.Now, test.events.Add)
 323         ctx, cancel := context.WithCancel(t.Context())
 324         done := make(chan error)
 325         go func() {
 326                 done <- test.loop.Run(ctx)
 327         }()
 328
 329         t.Cleanup(func() {
 330                 cancel()
 331                 require.True(t, errors.Is(<-done, context.Canceled))
 332                 require.NoError(t, memory.Close())
 333         })
 334
 335         return test
 336 }
 337
 338 func reasoningItem(id, encrypted string) map[string]any {
 339         return map[string]any{
 340                 "type":              "reasoning",
 341                 "id":                id,
 342                 "encrypted_content": encrypted,
 343                 "summary":           []any{},
 344         }
 345 }
 346
 347 func functionCallItem(id, callID, name, arguments string) map[string]any {
 348         return map[string]any{
 349                 "type":      "function_call",
 350                 "id":        id,
 351                 "call_id":   callID,
 352                 "name":      name,
 353                 "arguments": arguments,
 354         }
 355 }