git.giacomolli.uk - onechat/blob - subagent_run_test.go

   1 package main
   2
   3 import (
   4         "io"
   5         "net/http"
   6         "os"
   7         "path/filepath"
   8         "strconv"
   9         "strings"
  10         "testing"
  11         "time"
  12
  13         "github.com/stretchr/testify/require"
  14 )
  15
  16 // subagentTestFullContext is the error body of a call whose input does not fit
  17 // in the context of the model.
  18 const subagentTestFullContext = `{"error": {"message": "Your input exceeds the context window of this model.", "type": "invalid_request_error", "code": "context_length_exceeded"}}`
  19
  20 func TestSpawnRunsASubagentInTheBackground(t *testing.T) {
  21         server := newModelFake(t)
  22         var test *TurnTestLoop
  23         server.reply = func(w http.ResponseWriter, call ModelFakeCall) {
  24                 last := subagentTestLastItem(call)
  25                 switch {
  26                 case subagentTestCall(call) && last["type"] == "function_call_output":
  27                         // The report comes after the reply of the turn of the spawn.
  28                         test.events.WaitFor(t, "reply", 1)
  29                         modelWriteEvents(w, modelCompletedEvent([]any{modelMessageItem("There are 3 files.")}, 100, 0, 5))
  30                 case subagentTestCall(call):
  31                         modelWriteEvents(w, modelCompletedEvent([]any{modelFunctionCallItem("fc_1", "call_1", "shell", `{"command":"pwd"}`)}, 100, 0, 5))
  32                 case last["type"] == "function_call_output":
  33                         modelWriteEvents(w, modelCompletedEvent([]any{modelMessageItem("Started.")}, 100, 0, 5))
  34                 case strings.HasPrefix(subagentTestText(last), "["):
  35                         modelWriteEvents(w, modelCompletedEvent([]any{modelMessageItem("Noted.")}, 100, 0, 5))
  36                 default:
  37                         modelWriteEvents(w, modelCompletedEvent([]any{modelFunctionCallItem("fc_1", "call_1", "spawn", `{"tasks":["Count the files."],"effort":null}`)}, 100, 0, 5))
  38                 }
  39         }
  40
  41         test = newTestTurnLoop(t, server)
  42         _, err := test.loop.Submit(terminalSource, "Count the files in a subagent.")
  43         require.NoError(t, err)
  44         test.events.WaitFor(t, "idle", 2)
  45
  46         name := test.events.Last("subagent")
  47         require.True(t, subagentNameValid(name), name)
  48         work := filepath.Join(test.loop.runs.subagents.dir, name, "work")
  49         require.DirExists(t, work)
  50
  51         report := "[" + name + "]\nThere are 3 files.\n\nFull chat: zoom(\"" + name + "\")"
  52         require.Equal(t, []string{
  53                 "user: Count the files in a subagent.",
  54                 `tool: spawn {"tasks":["Count the files."],"effort":null}`,
  55                 "echo: Started " + name + ". Each one's report will reach you as a message when it finishes.",
  56                 "reply: Started.",
  57                 "work: " + report,
  58                 "reply: Noted.",
  59         }, test.Log(t))
  60
  61         var first, second *ModelFakeCall
  62         for n := range server.Calls() {
  63                 call := server.Call(n)
  64                 if !subagentTestCall(call) {
  65                         continue
  66                 }
  67
  68                 if first == nil {
  69                         first = &call
  70                 } else {
  71                         second = &call
  72                 }
  73         }
  74
  75         require.NotNil(t, second)
  76
  77         // The first request of the subagent holds the view, the state, and its
  78         // whole log, with its own prompt and tools.
  79         texts := turnInputTexts(t, *first)
  80         view := strings.Join(texts[:len(texts)-2], "")
  81         require.True(t, strings.HasPrefix(view, "<chat>\n0+1|user: Count the files in a subagent.\n1+1|tool: spawn"), view)
  82         require.Equal(t, "</chat>\n\nNow: Thursday 2026-01-01 12:00 +00:00", texts[len(texts)-2])
  83         require.Equal(t, "0|note: subagent\n\n1|user: Count the files.", texts[len(texts)-1])
  84
  85         instructions, err := test.agent.SubagentPrompt()
  86         require.NoError(t, err)
  87         require.Equal(t, instructions, first.body["instructions"])
  88
  89         var tools []string
  90         for _, tool := range first.body["tools"].([]any) {
  91                 tools = append(tools, tool.(map[string]any)["name"].(string))
  92         }
  93
  94         require.Equal(t, []string{"zoom", "date", "shell", "read", "write", "edit"}, tools)
  95         require.Equal(t, work+"\n", subagentTestLastItem(*second)["output"])
  96
  97         subagent, err := test.loop.runs.subagents.Get(t.Context(), name)
  98         require.NoError(t, err)
  99
 100         log, err := subagent.Show()
 101         require.NoError(t, err)
 102         require.Equal(t, "0|note: subagent\n\n1|user: Count the files.\n\n2|tool: shell {\"command\":\"pwd\"}\n\n3|echo: "+work+"\n\n\n4|reply: There are 3 files.", log)
 103
 104         var agents []string
 105         for _, record := range modelReadUsage(t, test.usagePath) {
 106                 agents = append(agents, record.Agent)
 107         }
 108
 109         require.ElementsMatch(t, []string{"", "", "", name, name}, agents)
 110 }
 111
 112 func TestReportJoinsTheNextRequestOfARunningTurn(t *testing.T) {
 113         server := newModelFake(t)
 114         arrived := make(chan struct{})
 115         release := make(chan struct{})
 116         server.reply = func(w http.ResponseWriter, call ModelFakeCall) {
 117                 if call.number == 0 {
 118                         close(arrived)
 119                         <-release
 120                         modelWriteEvents(w, modelCompletedEvent([]any{modelFunctionCallItem("fc_1", "call_1", "date", `{"id":0}`)}, 100, 0, 5))
 121
 122                         return
 123                 }
 124
 125                 modelWriteEvents(w, modelCompletedEvent([]any{modelMessageItem("done")}, 100, 0, 5))
 126         }
 127
 128         test := newTestTurnLoop(t, server)
 129         _, err := test.loop.Submit(terminalSource, "first")
 130         require.NoError(t, err)
 131
 132         <-arrived
 133         report := subagentReport("GreenOtter", []string{"It is done."})
 134         test.loop.Report(report, false)
 135         close(release)
 136
 137         test.events.WaitFor(t, "idle", 1)
 138         require.Equal(t, report, subagentTestText(subagentTestLastItem(server.Call(1))))
 139
 140         time.Sleep(50 * time.Millisecond)
 141         require.Equal(t, 2, server.Calls())
 142         require.Equal(t, 1, test.events.Count("busy"))
 143 }
 144
 145 func TestReportOfAStoppedSubagentStartsNoTurn(t *testing.T) {
 146         server := newModelFake(t)
 147         test := newTestTurnLoop(t, server)
 148         report := subagentReport("GreenOtter", []string{"failed: context canceled"})
 149         test.loop.Report(report, true)
 150
 151         test.events.WaitFor(t, "work", 1)
 152         time.Sleep(50 * time.Millisecond)
 153         require.Equal(t, []string{"work: " + report}, test.Log(t))
 154         require.Zero(t, server.Calls())
 155         require.Zero(t, test.events.Count("busy"))
 156 }
 157
 158 func TestFullContextGivesAFailedReport(t *testing.T) {
 159         server := newModelFake(t)
 160         server.reply = func(w http.ResponseWriter, call ModelFakeCall) {
 161                 if subagentTestCall(call) {
 162                         w.WriteHeader(http.StatusBadRequest)
 163                         _, _ = io.WriteString(w, subagentTestFullContext)
 164
 165                         return
 166                 }
 167
 168                 modelWriteEvents(w, modelCompletedEvent([]any{modelMessageItem("It failed.")}, 100, 0, 5))
 169         }
 170
 171         test := newTestTurnLoop(t, server)
 172         _, err := test.loop.runs.Spawn(t.Context(), []string{"Read the big file."}, nil)
 173         require.NoError(t, err)
 174         test.events.WaitFor(t, "idle", 1)
 175
 176         name := test.events.Last("subagent")
 177         require.Equal(t, []string{
 178                 "work: [" + name + "]\nfailed: model call: HTTP 400: " + subagentTestFullContext + "\n\nFull chat: zoom(\"" + name + "\")",
 179                 "reply: It failed.",
 180         }, test.Log(t))
 181 }
 182
 183 func TestSubagentHasItsOwnEffortAndNoSpawn(t *testing.T) {
 184         server := newModelFake(t)
 185         server.reply = func(w http.ResponseWriter, call ModelFakeCall) {
 186                 if subagentTestCall(call) && subagentTestLastItem(call)["type"] != "function_call_output" {
 187                         modelWriteEvents(w, modelCompletedEvent([]any{modelFunctionCallItem("fc_1", "call_1", "spawn", `{"tasks":["More."],"effort":null}`)}, 100, 0, 5))
 188
 189                         return
 190                 }
 191
 192                 modelWriteEvents(w, modelCompletedEvent([]any{modelMessageItem("ok")}, 100, 0, 5))
 193         }
 194
 195         test := newTestTurnLoop(t, server)
 196         runs := test.loop.runs
 197         _, err := runs.Spawn(t.Context(), nil, nil)
 198         require.EqualError(t, err, "no tasks")
 199
 200         effort := "max"
 201         _, err = runs.Spawn(t.Context(), []string{"Think."}, &effort)
 202         require.EqualError(t, err, `no effort "max": one of low, medium, high, xhigh`)
 203
 204         effort = "high"
 205         _, err = runs.Spawn(t.Context(), []string{"Think hard."}, &effort)
 206         require.NoError(t, err)
 207         test.events.WaitFor(t, "idle", 1)
 208
 209         for n := range server.Calls() {
 210                 call := server.Call(n)
 211                 if subagentTestCall(call) {
 212                         require.Equal(t, "high", call.body["reasoning"].(map[string]any)["effort"])
 213                 }
 214         }
 215
 216         subagent, err := runs.subagents.Get(t.Context(), test.events.Last("subagent"))
 217         require.NoError(t, err)
 218
 219         log, err := subagent.Show()
 220         require.NoError(t, err)
 221         require.Equal(t, "0|note: subagent at effort high\n\n1|user: Think hard.\n\n2|tool: spawn {\"tasks\":[\"More.\"],\"effort\":null}\n\n3|echo: error: unknown tool \"spawn\"\n\n4|reply: ok", log)
 222
 223         entries, err := os.ReadDir(runs.subagents.dir)
 224         require.NoError(t, err)
 225         require.Len(t, entries, 1)
 226 }
 227
 228 func TestTellReachesARunningSubagentBetweenItsToolCalls(t *testing.T) {
 229         server := newModelFake(t)
 230         arrived := make(chan struct{})
 231         told := make(chan struct{})
 232         server.reply = func(w http.ResponseWriter, call ModelFakeCall) {
 233                 switch {
 234                 case !subagentTestCall(call):
 235                         modelWriteEvents(w, modelCompletedEvent([]any{modelMessageItem("Noted.")}, 100, 0, 5))
 236                 case call.number == 0:
 237                         close(arrived)
 238                         <-told
 239                         modelWriteEvents(w, modelCompletedEvent([]any{modelFunctionCallItem("fc_1", "call_1", "shell", `{"command":"true"}`)}, 100, 0, 5))
 240                 default:
 241                         modelWriteEvents(w, modelCompletedEvent([]any{modelMessageItem("Used the second disk.")}, 100, 0, 5))
 242                 }
 243         }
 244
 245         test := newTestTurnLoop(t, server)
 246         runs := test.loop.runs
 247         _, err := runs.Spawn(t.Context(), []string{"Copy the files."}, nil)
 248         require.NoError(t, err)
 249
 250         <-arrived
 251         name := test.events.Last("subagent")
 252         output, err := runs.Tell(t.Context(), name, "Use the second disk.")
 253         require.NoError(t, err)
 254         require.Equal(t, "Sent. "+name+" reads it between its tool calls, and its report answers it.", output)
 255         close(told)
 256
 257         test.events.WaitFor(t, "idle", 1)
 258         require.Equal(t, "Use the second disk.", subagentTestText(subagentTestLastItem(server.Call(1))))
 259         require.Equal(t, "work: "+subagentReport(name, []string{"Used the second disk."}), test.Log(t)[0])
 260 }
 261
 262 func TestTellAfterTheFinalAnswerStartsAnotherChain(t *testing.T) {
 263         server := newModelFake(t)
 264         arrived := make(chan struct{})
 265         told := make(chan struct{})
 266         server.reply = func(w http.ResponseWriter, call ModelFakeCall) {
 267                 switch {
 268                 case !subagentTestCall(call):
 269                         modelWriteEvents(w, modelCompletedEvent([]any{modelMessageItem("Noted.")}, 100, 0, 5))
 270                 case call.number == 0:
 271                         close(arrived)
 272                         <-told
 273                         modelWriteEvents(w, modelCompletedEvent([]any{modelMessageItem("First.")}, 100, 0, 5))
 274                 default:
 275                         modelWriteEvents(w, modelCompletedEvent([]any{modelMessageItem("Second.")}, 100, 0, 5))
 276                 }
 277         }
 278
 279         test := newTestTurnLoop(t, server)
 280         runs := test.loop.runs
 281         _, err := runs.Spawn(t.Context(), []string{"Count."}, nil)
 282         require.NoError(t, err)
 283
 284         <-arrived
 285         name := test.events.Last("subagent")
 286         _, err = runs.Tell(t.Context(), name, "Count again.")
 287         require.NoError(t, err)
 288         close(told)
 289
 290         test.events.WaitFor(t, "idle", 1)
 291         require.Equal(t, "0|note: subagent\n\n1|user: Count.\n\n2|user: Count again.\n\n3|reply: First.", subagentTestText(subagentTestLastItem(server.Call(1))))
 292         require.Equal(t, "work: "+subagentReport(name, []string{"First.", "Second."}), test.Log(t)[0])
 293 }
 294
 295 func TestTellToAFinishedSubagentGoesOnFromItsLog(t *testing.T) {
 296         server := newModelFake(t)
 297         server.reply = func(w http.ResponseWriter, call ModelFakeCall) {
 298                 text := "Noted."
 299                 if subagentTestCall(call) {
 300                         text = "Reply " + strconv.Itoa(call.number) + "."
 301                 }
 302
 303                 modelWriteEvents(w, modelCompletedEvent([]any{modelMessageItem(text)}, 100, 0, 5))
 304         }
 305
 306         test := newTestTurnLoop(t, server)
 307         runs := test.loop.runs
 308         effort := "high"
 309         _, err := runs.Spawn(t.Context(), []string{"Count."}, &effort)
 310         require.NoError(t, err)
 311         test.events.WaitFor(t, "idle", 1)
 312
 313         name := test.events.Last("subagent")
 314         output, err := runs.Tell(t.Context(), name, "And the hidden files?")
 315         require.NoError(t, err)
 316         require.Equal(t, "Sent. "+name+"'s reply will reach you as a message.", output)
 317         test.events.WaitFor(t, "idle", 2)
 318
 319         call := server.Call(2)
 320         require.True(t, subagentTestCall(call))
 321         require.Equal(t, "high", call.body["reasoning"].(map[string]any)["effort"])
 322         require.Equal(t, "0|note: subagent at effort high\n\n1|user: Count.\n\n2|reply: Reply 0.\n\n3|user: And the hidden files?", subagentTestText(subagentTestLastItem(call)))
 323
 324         // The view of the new run holds the chat of now, with the first report.
 325         texts := turnInputTexts(t, call)
 326         require.Contains(t, strings.Join(texts[:len(texts)-2], ""), "work: ["+name+"]")
 327         require.Equal(t, "work: "+subagentReport(name, []string{"Reply 2."}), test.Log(t)[2])
 328
 329         _, err = runs.Tell(t.Context(), "BlueFox", "Hello.")
 330         require.EqualError(t, err, "no agent BlueFox")
 331 }
 332
 333 func TestStopEndsASubagentAndItsCommand(t *testing.T) {
 334         server := newModelFake(t)
 335         server.reply = func(w http.ResponseWriter, call ModelFakeCall) {
 336                 modelWriteEvents(w, modelCompletedEvent([]any{modelFunctionCallItem("fc_1", "call_1", "shell", `{"command":"touch started && sleep 30"}`)}, 100, 0, 5))
 337         }
 338
 339         test := newTestTurnLoop(t, server)
 340         runs := test.loop.runs
 341         _, err := runs.Spawn(t.Context(), []string{"Wait long."}, nil)
 342         require.NoError(t, err)
 343
 344         name := test.events.Last("subagent")
 345         subagent, err := runs.subagents.Get(t.Context(), name)
 346         require.NoError(t, err)
 347
 348         require.Eventually(t, func() bool {
 349                 _, err := os.Stat(filepath.Join(subagent.WorkDir(), "started"))
 350                 return err == nil
 351         }, 10*time.Second, time.Millisecond)
 352
 353         start := time.Now()
 354         found, err := test.loop.StopSubagent(name)
 355         require.NoError(t, err)
 356         require.True(t, found)
 357
 358         test.events.WaitFor(t, "work", 1)
 359         require.Less(t, time.Since(start), 10*time.Second)
 360         require.Equal(t, []string{
 361                 "note: Stopped by the user: " + name + ".",
 362                 "work: " + subagentReport(name, []string{"stopped"}),
 363         }, test.Log(t))
 364
 365         log, err := subagent.Show()
 366         require.NoError(t, err)
 367         require.Contains(t, log, "echo: \n[interrupted]")
 368
 369         found, err = test.loop.StopSubagent(name)
 370         require.NoError(t, err)
 371         require.False(t, found)
 372
 373         time.Sleep(50 * time.Millisecond)
 374         require.Zero(t, test.events.Count("busy"))
 375 }
 376
 377 func TestCloseEndsTheSubagentsWithNoReport(t *testing.T) {
 378         server := newModelFake(t)
 379         arrived := make(chan struct{})
 380         release := make(chan struct{})
 381         t.Cleanup(func() {
 382                 close(release)
 383         })
 384
 385         server.reply = func(w http.ResponseWriter, call ModelFakeCall) {
 386                 close(arrived)
 387                 <-release
 388         }
 389
 390         test := newTestTurnLoop(t, server)
 391         runs := test.loop.runs
 392         _, err := runs.Spawn(t.Context(), []string{"Wait."}, nil)
 393         require.NoError(t, err)
 394
 395         <-arrived
 396         runs.Close()
 397         require.Empty(t, runs.running)
 398         require.Empty(t, test.Log(t))
 399
 400         _, err = runs.Spawn(t.Context(), []string{"Again."}, nil)
 401         require.ErrorIs(t, err, errSubagentClosed)
 402
 403         entries, err := os.ReadDir(runs.subagents.dir)
 404         require.NoError(t, err)
 405         require.Len(t, entries, 1)
 406 }
 407
 408 // subagentTestCall tells if call is a call of a subagent.
 409 func subagentTestCall(call ModelFakeCall) bool {
 410         instructions, _ := call.body["instructions"].(string)
 411
 412         return strings.HasPrefix(instructions, "You are a subagent")
 413 }
 414
 415 // subagentTestLastItem returns the last input item of call.
 416 func subagentTestLastItem(call ModelFakeCall) map[string]any {
 417         input := call.body["input"].([]any)
 418
 419         return input[len(input)-1].(map[string]any)
 420 }
 421
 422 // subagentTestText returns the text of the user message item, or "" for
 423 // another item.
 424 func subagentTestText(item map[string]any) string {
 425         content, _ := item["content"].([]any)
 426         var text strings.Builder
 427         for _, part := range content {
 428                 value, _ := part.(map[string]any)["text"].(string)
 429                 text.WriteString(value)
 430         }
 431
 432         return text.String()
 433 }