X-Git-Url: https://git.giacomolli.uk/onechat/blobdiff_plain/ddd0a50f7b801112376d7fb27bfd05d2fc26c2fd..effd32b0f2e4d146b219b1727601b0b7f6593638:/subagent_run_test.go?ds=inline diff --git a/subagent_run_test.go b/subagent_run_test.go index 3027fe6..c5ad299 100644 --- a/subagent_run_test.go +++ b/subagent_run_test.go @@ -1,11 +1,15 @@ package main import ( + "context" "io" + "log/slog" "net/http" "os" "path/filepath" + "strconv" "strings" + "sync/atomic" "testing" "time" @@ -18,7 +22,9 @@ const subagentTestFullContext = `{"error": {"message": "Your input exceeds the c func TestSpawnRunsASubagentInTheBackground(t *testing.T) { server := newModelFake(t) + var test *TurnTestLoop + server.reply = func(w http.ResponseWriter, call ModelFakeCall) { last := subagentTestLastItem(call) switch { @@ -44,6 +50,7 @@ func TestSpawnRunsASubagentInTheBackground(t *testing.T) { name := test.events.Last("subagent") require.True(t, subagentNameValid(name), name) + work := filepath.Join(test.loop.runs.subagents.dir, name, "work") require.DirExists(t, work) @@ -56,8 +63,17 @@ func TestSpawnRunsASubagentInTheBackground(t *testing.T) { "work: " + report, "reply: Noted.", }, test.Log(t)) + require.Equal(t, Event{ + Kind: "work", + ID: 4, + Text: report, + Source: name, + Status: subagentFinished, + }, test.events.LastEvent("work")) + require.Equal(t, "<6>subagent started name="+name+"\n<6>subagent done name="+name+" after=0s\n", test.logs.String()) var first, second *ModelFakeCall + for n := range server.Calls() { call := server.Call(n) if !subagentTestCall(call) { @@ -86,6 +102,7 @@ func TestSpawnRunsASubagentInTheBackground(t *testing.T) { require.Equal(t, instructions, first.body["instructions"]) var tools []string + for _, tool := range first.body["tools"].([]any) { tools = append(tools, tool.(map[string]any)["name"].(string)) } @@ -101,6 +118,7 @@ func TestSpawnRunsASubagentInTheBackground(t *testing.T) { require.Equal(t, "0|note: subagent\n\n1|user: Count the files.\n\n2|tool: shell {\"command\":\"pwd\"}\n\n3|echo: "+work+"\n\n\n4|reply: There are 3 files.", log) var agents []string + for _, record := range modelReadUsage(t, test.usagePath) { agents = append(agents, record.Agent) } @@ -129,8 +147,9 @@ func TestReportJoinsTheNextRequestOfARunningTurn(t *testing.T) { require.NoError(t, err) <-arrived + report := subagentReport("GreenOtter", []string{"It is done."}) - test.loop.Report(report, false) + test.loop.Report("GreenOtter", report, subagentFinished) close(release) test.events.WaitFor(t, "idle", 1) @@ -145,7 +164,7 @@ func TestReportOfAStoppedSubagentStartsNoTurn(t *testing.T) { server := newModelFake(t) test := newTestTurnLoop(t, server) report := subagentReport("GreenOtter", []string{"failed: context canceled"}) - test.loop.Report(report, true) + test.loop.Report("GreenOtter", report, subagentStopped) test.events.WaitFor(t, "work", 1) time.Sleep(50 * time.Millisecond) @@ -159,6 +178,7 @@ func TestFullContextGivesAFailedReport(t *testing.T) { server.reply = func(w http.ResponseWriter, call ModelFakeCall) { if subagentTestCall(call) { w.WriteHeader(http.StatusBadRequest) + _, _ = io.WriteString(w, subagentTestFullContext) return @@ -174,9 +194,11 @@ func TestFullContextGivesAFailedReport(t *testing.T) { name := test.events.Last("subagent") require.Equal(t, []string{ - "work: [" + name + "]\nfailed: model call: HTTP 400: " + subagentTestFullContext + "\n\nFull chat: zoom(\"" + name + "\")", + "work: [" + name + "]\nfailed: create model response: model call: HTTP 400: " + subagentTestFullContext + "\n\nFull chat: zoom(\"" + name + "\")", "reply: It failed.", }, test.Log(t)) + require.Equal(t, subagentFailed, test.events.LastEvent("work").Status) + require.Equal(t, "<6>subagent started name="+name+"\n<3>subagent failed name="+name+" after=0s: create model response: model call: HTTP 400: "+subagentTestFullContext+"\n", test.logs.String()) } func TestSubagentHasItsOwnEffortAndNoSpawn(t *testing.T) { @@ -224,6 +246,159 @@ func TestSubagentHasItsOwnEffortAndNoSpawn(t *testing.T) { require.Len(t, entries, 1) } +func TestTellReachesARunningSubagentBetweenItsToolCalls(t *testing.T) { + server := newModelFake(t) + arrived := make(chan struct{}) + told := make(chan struct{}) + server.reply = func(w http.ResponseWriter, call ModelFakeCall) { + switch { + case !subagentTestCall(call): + modelWriteEvents(w, modelCompletedEvent([]any{modelMessageItem("Noted.")}, 100, 0, 5)) + case call.number == 0: + close(arrived) + <-told + modelWriteEvents(w, modelCompletedEvent([]any{modelFunctionCallItem("fc_1", "call_1", "shell", `{"command":"true"}`)}, 100, 0, 5)) + default: + modelWriteEvents(w, modelCompletedEvent([]any{modelMessageItem("Used the second disk.")}, 100, 0, 5)) + } + } + + test := newTestTurnLoop(t, server) + runs := test.loop.runs + _, err := runs.Spawn(t.Context(), []string{"Copy the files."}, nil) + require.NoError(t, err) + + <-arrived + + name := test.events.Last("subagent") + output, err := runs.Tell(t.Context(), name, "Use the second disk.") + require.NoError(t, err) + require.Equal(t, "Sent. "+name+" reads it between its tool calls, and its report answers it.", output) + close(told) + + test.events.WaitFor(t, "idle", 1) + require.Equal(t, "Use the second disk.", subagentTestText(subagentTestLastItem(server.Call(1)))) + require.Equal(t, "work: "+subagentReport(name, []string{"Used the second disk."}), test.Log(t)[0]) +} + +func TestTellAfterTheFinalAnswerStartsAnotherChain(t *testing.T) { + server := newModelFake(t) + arrived := make(chan struct{}) + told := make(chan struct{}) + server.reply = func(w http.ResponseWriter, call ModelFakeCall) { + switch { + case !subagentTestCall(call): + modelWriteEvents(w, modelCompletedEvent([]any{modelMessageItem("Noted.")}, 100, 0, 5)) + case call.number == 0: + close(arrived) + <-told + modelWriteEvents(w, modelCompletedEvent([]any{modelMessageItem("First.")}, 100, 0, 5)) + default: + modelWriteEvents(w, modelCompletedEvent([]any{modelMessageItem("Second.")}, 100, 0, 5)) + } + } + + test := newTestTurnLoop(t, server) + runs := test.loop.runs + _, err := runs.Spawn(t.Context(), []string{"Count."}, nil) + require.NoError(t, err) + + <-arrived + + name := test.events.Last("subagent") + _, err = runs.Tell(t.Context(), name, "Count again.") + require.NoError(t, err) + close(told) + + test.events.WaitFor(t, "idle", 1) + require.Equal(t, "0|note: subagent\n\n1|user: Count.\n\n2|user: Count again.\n\n3|reply: First.", subagentTestText(subagentTestLastItem(server.Call(1)))) + require.Equal(t, "work: "+subagentReport(name, []string{"First.", "Second."}), test.Log(t)[0]) +} + +func TestTellToAFinishedSubagentGoesOnFromItsLog(t *testing.T) { + server := newModelFake(t) + server.reply = func(w http.ResponseWriter, call ModelFakeCall) { + text := "Noted." + if subagentTestCall(call) { + text = "Reply " + strconv.Itoa(call.number) + "." + } + + modelWriteEvents(w, modelCompletedEvent([]any{modelMessageItem(text)}, 100, 0, 5)) + } + + test := newTestTurnLoop(t, server) + runs := test.loop.runs + effort := "high" + _, err := runs.Spawn(t.Context(), []string{"Count."}, &effort) + require.NoError(t, err) + test.events.WaitFor(t, "idle", 1) + + name := test.events.Last("subagent") + output, err := runs.Tell(t.Context(), name, "And the hidden files?") + require.NoError(t, err) + require.Equal(t, "Sent. "+name+"'s reply will reach you as a message.", output) + test.events.WaitFor(t, "idle", 2) + + call := server.Call(2) + require.True(t, subagentTestCall(call)) + require.Equal(t, "high", call.body["reasoning"].(map[string]any)["effort"]) + require.Equal(t, "0|note: subagent at effort high\n\n1|user: Count.\n\n2|reply: Reply 0.\n\n3|user: And the hidden files?", subagentTestText(subagentTestLastItem(call))) + + // The view of the new run holds the chat of now, with the first report. + texts := turnInputTexts(t, call) + require.Contains(t, strings.Join(texts[:len(texts)-2], ""), "work: ["+name+"]") + require.Equal(t, "work: "+subagentReport(name, []string{"Reply 2."}), test.Log(t)[2]) + + _, err = runs.Tell(t.Context(), "BlueFox", "Hello.") + require.EqualError(t, err, "no agent BlueFox") +} + +func TestStopEndsASubagentAndItsCommand(t *testing.T) { + server := newModelFake(t) + server.reply = func(w http.ResponseWriter, _ ModelFakeCall) { + modelWriteEvents(w, modelCompletedEvent([]any{modelFunctionCallItem("fc_1", "call_1", "shell", `{"command":"touch started && sleep 30"}`)}, 100, 0, 5)) + } + + test := newTestTurnLoop(t, server) + runs := test.loop.runs + _, err := runs.Spawn(t.Context(), []string{"Wait long."}, nil) + require.NoError(t, err) + + name := test.events.Last("subagent") + subagent, err := runs.subagents.Get(t.Context(), name) + require.NoError(t, err) + + require.Eventually(t, func() bool { + _, err := os.Stat(filepath.Join(subagent.WorkDir(), "started")) + return err == nil + }, 10*time.Second, time.Millisecond) + + start := time.Now() + found, err := test.loop.StopSubagent(name) + require.NoError(t, err) + require.True(t, found) + + test.events.WaitFor(t, "work", 1) + require.Less(t, time.Since(start), 10*time.Second) + require.Equal(t, []string{ + "note: Stopped by the user: " + name + ".", + "work: " + subagentReport(name, []string{"stopped"}), + }, test.Log(t)) + require.Equal(t, subagentStopped, test.events.LastEvent("work").Status) + require.Equal(t, "<6>subagent started name="+name+"\n<6>subagent stopped name="+name+" after=0s\n", test.logs.String()) + + log, err := subagent.Show() + require.NoError(t, err) + require.Contains(t, log, "echo: \n[interrupted]") + + found, err = test.loop.StopSubagent(name) + require.NoError(t, err) + require.False(t, found) + + time.Sleep(50 * time.Millisecond) + require.Zero(t, test.events.Count("busy")) +} + func TestCloseEndsTheSubagentsWithNoReport(t *testing.T) { server := newModelFake(t) arrived := make(chan struct{}) @@ -232,7 +407,7 @@ func TestCloseEndsTheSubagentsWithNoReport(t *testing.T) { close(release) }) - server.reply = func(w http.ResponseWriter, call ModelFakeCall) { + server.reply = func(_ http.ResponseWriter, _ ModelFakeCall) { close(arrived) <-release } @@ -255,6 +430,149 @@ func TestCloseEndsTheSubagentsWithNoReport(t *testing.T) { require.Len(t, entries, 1) } +func TestRestartResumesTheCutSubagents(t *testing.T) { + server := newModelFake(t) + arrived := make(chan struct{}, 1) + release := make(chan struct{}) + t.Cleanup(func() { + close(release) + }) + + var restarted atomic.Bool + + server.reply = func(w http.ResponseWriter, call ModelFakeCall) { + switch { + case subagentTestCall(call) && !restarted.Load(): + arrived <- struct{}{} + + <-release + case subagentTestCall(call): + modelWriteEvents(w, modelCompletedEvent([]any{modelMessageItem("Built.")}, 100, 0, 5)) + default: + modelWriteEvents(w, modelCompletedEvent([]any{modelMessageItem("Noted.")}, 100, 0, 5)) + } + } + + agent, err := openAgent(t.TempDir()) + require.NoError(t, err) + + first := &TurnEventRecorder{} + loop, _, stop := subagentTestStart(t, agent, server, first, discardLog()) + _, err = loop.runs.Spawn(t.Context(), []string{"Build the project."}, nil) + require.NoError(t, err) + + <-arrived + + name := first.Last("subagent") + + stop() + require.Zero(t, first.Count("work")) + + data, err := os.ReadFile(filepath.Join(agent.dir, subagentWorkFileName)) + require.NoError(t, err) + require.JSONEq(t, `{"running": ["`+name+`"]}`, string(data)) + + restarted.Store(true) + + second := &TurnEventRecorder{} + loop, memory, stop := subagentTestStart(t, agent, server, second, discardLog()) + defer stop() + + second.WaitFor(t, "idle", 1) + + note := "The daemon restarted, cutting " + name + "; nothing was lost: go on." + log := subagentTestChat(t, memory) + require.Equal(t, []string{ + "note: " + note, + "work: " + subagentReport(name, []string{"Built."}), + "reply: Noted.", + }, log[len(log)-3:]) + + subagent, err := loop.runs.subagents.Get(t.Context(), name) + require.NoError(t, err) + + text, err := subagent.Show() + require.NoError(t, err) + require.Equal(t, "0|note: subagent\n\n1|user: Build the project.\n\n2|user: "+note+"\n\n3|reply: Built.", text) + + data, err = os.ReadFile(filepath.Join(agent.dir, subagentWorkFileName)) + require.NoError(t, err) + require.JSONEq(t, `{"running": []}`, string(data)) +} + +func TestRestartReportsASubagentThatCannotGoOn(t *testing.T) { + server := newModelFake(t) + agent, err := openAgent(t.TempDir()) + require.NoError(t, err) + require.NoError(t, os.WriteFile(filepath.Join(agent.dir, subagentWorkFileName), []byte(`{"running": ["BlueFox"]}`), 0o600)) + + events := &TurnEventRecorder{} + logs := &SyncBuffer{} + _, memory, stop := subagentTestStart(t, agent, server, events, daemonTestLogger(logs)) + defer stop() + + events.WaitFor(t, "idle", 1) + require.Equal(t, []string{ + "note: The daemon restarted, cutting BlueFox; nothing was lost: go on.", + "work: " + subagentReport("BlueFox", []string{"failed: no agent BlueFox"}), + "reply: a line", + }, subagentTestChat(t, memory)) + require.Equal(t, subagentFailed, events.LastEvent("work").Status) + require.Equal(t, "<3>subagent lost at restart name=BlueFox: no agent BlueFox\n", logs.String()) + + data, err := os.ReadFile(filepath.Join(agent.dir, subagentWorkFileName)) + require.NoError(t, err) + require.JSONEq(t, `{"running": []}`, string(data)) +} + +// subagentTestStart runs a loop of agent with its memory and subagents in the +// agent directory, as the daemon does. The returned function stops the loop +// and closes the files. +func subagentTestStart(t *testing.T, agent *Agent, server *ModelFake, events *TurnEventRecorder, logger *slog.Logger) (*TurnLoop, *Memory, func()) { + t.Helper() + + agent.config.Workdir = t.TempDir() + clock := newTestClock() + memory, err := openMemory(t.Context(), agent.dir, clock.Now, newFakeSummarizer(), discardLog()) + require.NoError(t, err) + + subagents := newSubagents(filepath.Join(agent.dir, subagentDirName), clock.Now, subagentNameRandom) + client := newModelClient(server.URL, ModelAPIKey(modelTestKey), agent.CacheKey(), filepath.Join(agent.dir, "usage.jsonl")) + whatsapp, _, _ := whatsappStartTest(t, t.TempDir(), clock) + loop, err := newTurnLoop(agent, memory, subagents, whatsapp, newTestSchedules(t, clock.Now), client, clock.Now, logger, events.Add) + require.NoError(t, err) + + ctx, cancel := context.WithCancel(t.Context()) + done := make(chan error) + go func() { + done <- loop.Run(ctx) + }() + + return loop, memory, func() { + cancel() + require.ErrorIs(t, <-done, context.Canceled) + require.NoError(t, subagents.Close()) + require.NoError(t, memory.Close()) + } +} + +// subagentTestChat returns each message of the chat in memory as +// "kind: text". +func subagentTestChat(t *testing.T, memory *Memory) []string { + t.Helper() + + var lines []string + + for id := range memory.Len() { + message, err := memory.Message(id) + require.NoError(t, err) + + lines = append(lines, message.Kind+": "+message.Text) + } + + return lines +} + // subagentTestCall tells if call is a call of a subagent. func subagentTestCall(call ModelFakeCall) bool { instructions, _ := call.body["instructions"].(string) @@ -273,7 +591,9 @@ func subagentTestLastItem(call ModelFakeCall) map[string]any { // another item. func subagentTestText(item map[string]any) string { content, _ := item["content"].([]any) + var text strings.Builder + for _, part := range content { value, _ := part.(map[string]any)["text"].(string) text.WriteString(value)