X-Git-Url: https://git.giacomolli.uk/onechat/blobdiff_plain/47f69638bc96100f2e446f63c7c3689023156e64..4c199e7d93b6a4529ed5956807911b5f6c5489a7:/subagent_run_test.go diff --git a/subagent_run_test.go b/subagent_run_test.go index 1fd6a4a..f637dcf 100644 --- a/subagent_run_test.go +++ b/subagent_run_test.go @@ -1,12 +1,14 @@ package main import ( + "context" "io" "net/http" "os" "path/filepath" "strconv" "strings" + "sync/atomic" "testing" "time" @@ -19,7 +21,9 @@ const subagentTestFullContext = `{"error": {"message": "Your input exceeds the c func TestSpawnRunsASubagentInTheBackground(t *testing.T) { server := newModelFake(t) + var test *TurnTestLoop + server.reply = func(w http.ResponseWriter, call ModelFakeCall) { last := subagentTestLastItem(call) switch { @@ -45,6 +49,7 @@ func TestSpawnRunsASubagentInTheBackground(t *testing.T) { name := test.events.Last("subagent") require.True(t, subagentNameValid(name), name) + work := filepath.Join(test.loop.runs.subagents.dir, name, "work") require.DirExists(t, work) @@ -57,8 +62,17 @@ func TestSpawnRunsASubagentInTheBackground(t *testing.T) { "work: " + report, "reply: Noted.", }, test.Log(t)) + require.Equal(t, Event{ + Kind: "work", + ID: 4, + Text: report, + Source: name, + Status: subagentFinished, + }, test.events.LastEvent("work")) + require.Equal(t, "subagent "+name+" started\nsubagent "+name+" done after 0s\n", test.logs.String()) var first, second *ModelFakeCall + for n := range server.Calls() { call := server.Call(n) if !subagentTestCall(call) { @@ -87,6 +101,7 @@ func TestSpawnRunsASubagentInTheBackground(t *testing.T) { require.Equal(t, instructions, first.body["instructions"]) var tools []string + for _, tool := range first.body["tools"].([]any) { tools = append(tools, tool.(map[string]any)["name"].(string)) } @@ -102,6 +117,7 @@ func TestSpawnRunsASubagentInTheBackground(t *testing.T) { require.Equal(t, "0|note: subagent\n\n1|user: Count the files.\n\n2|tool: shell {\"command\":\"pwd\"}\n\n3|echo: "+work+"\n\n\n4|reply: There are 3 files.", log) var agents []string + for _, record := range modelReadUsage(t, test.usagePath) { agents = append(agents, record.Agent) } @@ -130,8 +146,9 @@ func TestReportJoinsTheNextRequestOfARunningTurn(t *testing.T) { require.NoError(t, err) <-arrived + report := subagentReport("GreenOtter", []string{"It is done."}) - test.loop.Report(report, false) + test.loop.Report("GreenOtter", report, subagentFinished) close(release) test.events.WaitFor(t, "idle", 1) @@ -146,7 +163,7 @@ func TestReportOfAStoppedSubagentStartsNoTurn(t *testing.T) { server := newModelFake(t) test := newTestTurnLoop(t, server) report := subagentReport("GreenOtter", []string{"failed: context canceled"}) - test.loop.Report(report, true) + test.loop.Report("GreenOtter", report, subagentStopped) test.events.WaitFor(t, "work", 1) time.Sleep(50 * time.Millisecond) @@ -160,6 +177,7 @@ func TestFullContextGivesAFailedReport(t *testing.T) { server.reply = func(w http.ResponseWriter, call ModelFakeCall) { if subagentTestCall(call) { w.WriteHeader(http.StatusBadRequest) + _, _ = io.WriteString(w, subagentTestFullContext) return @@ -175,9 +193,11 @@ func TestFullContextGivesAFailedReport(t *testing.T) { name := test.events.Last("subagent") require.Equal(t, []string{ - "work: [" + name + "]\nfailed: model call: HTTP 400: " + subagentTestFullContext + "\n\nFull chat: zoom(\"" + name + "\")", + "work: [" + name + "]\nfailed: create model response: model call: HTTP 400: " + subagentTestFullContext + "\n\nFull chat: zoom(\"" + name + "\")", "reply: It failed.", }, test.Log(t)) + require.Equal(t, subagentFailed, test.events.LastEvent("work").Status) + require.Equal(t, "subagent "+name+" started\nsubagent "+name+" failed after 0s: create model response: model call: HTTP 400: "+subagentTestFullContext+"\n", test.logs.String()) } func TestSubagentHasItsOwnEffortAndNoSpawn(t *testing.T) { @@ -248,6 +268,7 @@ func TestTellReachesARunningSubagentBetweenItsToolCalls(t *testing.T) { require.NoError(t, err) <-arrived + name := test.events.Last("subagent") output, err := runs.Tell(t.Context(), name, "Use the second disk.") require.NoError(t, err) @@ -282,6 +303,7 @@ func TestTellAfterTheFinalAnswerStartsAnotherChain(t *testing.T) { require.NoError(t, err) <-arrived + name := test.events.Last("subagent") _, err = runs.Tell(t.Context(), name, "Count again.") require.NoError(t, err) @@ -332,7 +354,7 @@ func TestTellToAFinishedSubagentGoesOnFromItsLog(t *testing.T) { func TestStopEndsASubagentAndItsCommand(t *testing.T) { server := newModelFake(t) - server.reply = func(w http.ResponseWriter, call ModelFakeCall) { + server.reply = func(w http.ResponseWriter, _ ModelFakeCall) { modelWriteEvents(w, modelCompletedEvent([]any{modelFunctionCallItem("fc_1", "call_1", "shell", `{"command":"touch started && sleep 30"}`)}, 100, 0, 5)) } @@ -361,6 +383,8 @@ func TestStopEndsASubagentAndItsCommand(t *testing.T) { "note: Stopped by the user: " + name + ".", "work: " + subagentReport(name, []string{"stopped"}), }, test.Log(t)) + require.Equal(t, subagentStopped, test.events.LastEvent("work").Status) + require.Equal(t, "subagent "+name+" started\nsubagent "+name+" stopped after 0s\n", test.logs.String()) log, err := subagent.Show() require.NoError(t, err) @@ -382,7 +406,7 @@ func TestCloseEndsTheSubagentsWithNoReport(t *testing.T) { close(release) }) - server.reply = func(w http.ResponseWriter, call ModelFakeCall) { + server.reply = func(_ http.ResponseWriter, _ ModelFakeCall) { close(arrived) <-release } @@ -405,6 +429,147 @@ func TestCloseEndsTheSubagentsWithNoReport(t *testing.T) { require.Len(t, entries, 1) } +func TestRestartResumesTheCutSubagents(t *testing.T) { + server := newModelFake(t) + arrived := make(chan struct{}, 1) + release := make(chan struct{}) + t.Cleanup(func() { + close(release) + }) + + var restarted atomic.Bool + + server.reply = func(w http.ResponseWriter, call ModelFakeCall) { + switch { + case subagentTestCall(call) && !restarted.Load(): + arrived <- struct{}{} + + <-release + case subagentTestCall(call): + modelWriteEvents(w, modelCompletedEvent([]any{modelMessageItem("Built.")}, 100, 0, 5)) + default: + modelWriteEvents(w, modelCompletedEvent([]any{modelMessageItem("Noted.")}, 100, 0, 5)) + } + } + + agent, err := openAgent(t.TempDir()) + require.NoError(t, err) + + first := &TurnEventRecorder{} + loop, _, stop := subagentTestStart(t, agent, server, first) + _, err = loop.runs.Spawn(t.Context(), []string{"Build the project."}, nil) + require.NoError(t, err) + + <-arrived + + name := first.Last("subagent") + + stop() + require.Zero(t, first.Count("work")) + + data, err := os.ReadFile(filepath.Join(agent.dir, subagentWorkFileName)) + require.NoError(t, err) + require.JSONEq(t, `{"running": ["`+name+`"]}`, string(data)) + + restarted.Store(true) + + second := &TurnEventRecorder{} + loop, memory, stop := subagentTestStart(t, agent, server, second) + defer stop() + + second.WaitFor(t, "idle", 1) + + note := "The daemon restarted, cutting " + name + "; nothing was lost: go on." + log := subagentTestChat(t, memory) + require.Equal(t, []string{ + "note: " + note, + "work: " + subagentReport(name, []string{"Built."}), + "reply: Noted.", + }, log[len(log)-3:]) + + subagent, err := loop.runs.subagents.Get(t.Context(), name) + require.NoError(t, err) + + text, err := subagent.Show() + require.NoError(t, err) + require.Equal(t, "0|note: subagent\n\n1|user: Build the project.\n\n2|user: "+note+"\n\n3|reply: Built.", text) + + data, err = os.ReadFile(filepath.Join(agent.dir, subagentWorkFileName)) + require.NoError(t, err) + require.JSONEq(t, `{"running": []}`, string(data)) +} + +func TestRestartReportsASubagentThatCannotGoOn(t *testing.T) { + server := newModelFake(t) + agent, err := openAgent(t.TempDir()) + require.NoError(t, err) + require.NoError(t, os.WriteFile(filepath.Join(agent.dir, subagentWorkFileName), []byte(`{"running": ["BlueFox"]}`), 0o600)) + + events := &TurnEventRecorder{} + _, memory, stop := subagentTestStart(t, agent, server, events) + defer stop() + + events.WaitFor(t, "idle", 1) + require.Equal(t, []string{ + "note: The daemon restarted, cutting BlueFox; nothing was lost: go on.", + "work: " + subagentReport("BlueFox", []string{"failed: no agent BlueFox"}), + "reply: a line", + }, subagentTestChat(t, memory)) + require.Equal(t, subagentFailed, events.LastEvent("work").Status) + + data, err := os.ReadFile(filepath.Join(agent.dir, subagentWorkFileName)) + require.NoError(t, err) + require.JSONEq(t, `{"running": []}`, string(data)) +} + +// subagentTestStart runs a loop of agent with its memory and subagents in the +// agent directory, as the daemon does. The returned function stops the loop +// and closes the files. +func subagentTestStart(t *testing.T, agent *Agent, server *ModelFake, events *TurnEventRecorder) (*TurnLoop, *Memory, func()) { + t.Helper() + + agent.config.Workdir = t.TempDir() + clock := newTestClock() + memory, err := openMemory(t.Context(), agent.dir, clock.Now, newFakeSummarizer(), discardLog()) + require.NoError(t, err) + + subagents := newSubagents(filepath.Join(agent.dir, subagentDirName), clock.Now, subagentNameRandom) + client := newModelClient(server.URL, ModelAPIKey(modelTestKey), agent.CacheKey(), filepath.Join(agent.dir, "usage.jsonl")) + whatsapp, _, _ := whatsappStartTest(t, t.TempDir(), clock) + loop, err := newTurnLoop(agent, memory, subagents, whatsapp, newTestSchedules(t, clock.Now), client, clock.Now, discardLog(), events.Add) + require.NoError(t, err) + + ctx, cancel := context.WithCancel(t.Context()) + done := make(chan error) + go func() { + done <- loop.Run(ctx) + }() + + return loop, memory, func() { + cancel() + require.ErrorIs(t, <-done, context.Canceled) + require.NoError(t, subagents.Close()) + require.NoError(t, memory.Close()) + } +} + +// subagentTestChat returns each message of the chat in memory as +// "kind: text". +func subagentTestChat(t *testing.T, memory *Memory) []string { + t.Helper() + + var lines []string + + for id := range memory.Len() { + message, err := memory.Message(id) + require.NoError(t, err) + + lines = append(lines, message.Kind+": "+message.Text) + } + + return lines +} + // subagentTestCall tells if call is a call of a subagent. func subagentTestCall(call ModelFakeCall) bool { instructions, _ := call.body["instructions"].(string) @@ -423,7 +588,9 @@ func subagentTestLastItem(call ModelFakeCall) map[string]any { // another item. func subagentTestText(item map[string]any) string { content, _ := item["content"].([]any) + var text strings.Builder + for _, part := range content { value, _ := part.(map[string]any)["text"].(string) text.WriteString(value)