From: Anderson A. Giacomolli Date: Tue, 10 Nov 2026 14:42:27 +0000 (-0300) Subject: Carry new images through model request chains X-Git-Url: http://git.giacomolli.uk/onechat/commitdiff_plain/4343ca5fb6cc86eb887951f4fe16063bab29a238 Carry new images through model request chains Attach stored pixels to new user messages for the whole turn chain. Later turns keep text markers and descriptions, which keeps the request prefix stable. Leaf compaction sees pixels once per build and preserves visible details. Single-message zoom returns text and image parts while the durable log keeps only text. --- diff --git a/build_run_test.go b/build_run_test.go index b45745e..ef9c3c0 100644 --- a/build_run_test.go +++ b/build_run_test.go @@ -215,7 +215,7 @@ func newTestBuildRun(t *testing.T, server *ModelFake) *BuildRun { root := t.TempDir() usagePath := filepath.Join(root, "usage.jsonl") client := newModelClient(server.URL, ModelAPIKey(modelTestKey), "onechat-"+probeName, usagePath) - memory, err := openMemory(t.Context(), root, newTestClock().Now, newCompactor(client, probePrompt, defaultConfig().Compactor, promptTools()), discardLog()) + memory, err := openMemory(t.Context(), root, newTestClock().Now, newCompactor(client, probePrompt, defaultConfig().Compactor, promptTools(), &ImageStore{dir: t.TempDir()}), discardLog()) require.NoError(t, err) return &BuildRun{ diff --git a/chain.go b/chain.go index c4a4302..23c7dec 100644 --- a/chain.go +++ b/chain.go @@ -32,6 +32,7 @@ type Chain struct { tools *Toolbox request ModelRequest take func() []Message + images *ImageStore } // Run runs the chain from input, the input of its first request, and returns @@ -122,7 +123,12 @@ func (c *Chain) handle(ctx context.Context, input []any, response ModelResponse) input = append(input, outputs...) for _, message := range c.take() { - input = append(input, modelUserMessage(message.Text)) + item, err := imageUserMessage(c.images, message.Text) + if err != nil { + return nil, "", fmt.Errorf("read queued images: %w", err) + } + + input = append(input, item) } return input, "", nil @@ -141,7 +147,7 @@ type ChainFunctionCall struct { type ChainFunctionCallOutput struct { Type string `json:"type"` CallID string `json:"call_id"` - Output string `json:"output"` + Output any `json:"output"` } // chainInputItem returns an output item as an input item for the next request. diff --git a/compactor.go b/compactor.go index 8a16c6e..2b23901 100644 --- a/compactor.go +++ b/compactor.go @@ -4,7 +4,6 @@ import ( "context" "errors" "fmt" - "slices" "strings" "unicode/utf8" ) @@ -20,12 +19,13 @@ var compactorRuler = strings.Repeat("-", nodeBytes) // client with model. instructions gives the system prompt of the agent at each // build. tools are the tools of that agent, so a build sends the same prompt // and tools as its turns. -func newCompactor(client *ModelClient, instructions func() (string, error), model ModelConfig, tools []ModelTool) *Compactor { +func newCompactor(client *ModelClient, instructions func() (string, error), model ModelConfig, tools []ModelTool, images *ImageStore) *Compactor { return &Compactor{ client: client, model: model, instructions: instructions, tools: tools, + images: images, } } @@ -37,17 +37,19 @@ func newCompactor(client *ModelClient, instructions func() (string, error), mode // // When the line is over nodeBytes, the next try sends the same message, plus // one more user message with a note with the size of the newest line and its -// cut at the limit. A try never sends the model's earlier reply or reasoning -// back, so every try begins with the same view and task and has about the same -// size. After at most compactorTries tries, the shortest line is the node. A -// few bytes over the limit is fine, because the view measures the real sizes. A -// call that fails before any line fails the build, and the core tries it again -// at the next message. +// cut at the limit. Only the first call of an image leaf sends its pixels. A +// try never sends the model's earlier reply or reasoning back, so every try +// begins with the same view and task and has about the same size. After at most +// compactorTries tries, the shortest line is the node. A few bytes over the +// limit is fine, because the view measures the real sizes. A call that fails +// before any line fails the build, and the core tries it again at the next +// message. type Compactor struct { client *ModelClient model ModelConfig instructions func() (string, error) tools []ModelTool + images *ImageStore } // Summarize returns the line of the node of task. @@ -62,7 +64,13 @@ func (c *Compactor) Summarize(ctx context.Context, task Task) (string, error) { return "", err } - first := []any{modelUserMessage(task.Context + "\n" + text)} + source := task.Context + "\n" + text + parts, err := imageParts(c.images, source, task.Images) + if err != nil { + return "", fmt.Errorf("read compaction images: %w", err) + } + + first := []any{modelUserParts(parts)} var lines []string @@ -70,7 +78,7 @@ func (c *Compactor) Summarize(ctx context.Context, task Task) (string, error) { for len(lines) < compactorTries { input := first if note != "" { - input = append(slices.Clip(first), modelUserMessage(note)) + input = []any{modelUserMessage(source), modelUserMessage(note)} } var response ModelResponse @@ -111,12 +119,17 @@ func (c *Compactor) Summarize(ctx context.Context, task Task) (string, error) { // compactorTask returns the task of a build, with its input. func compactorTask(task Task) (string, error) { if task.Level == 0 && len(task.Input) == 1 { - return fmt.Sprintf(`Compaction: compress message %d into one line of at most %d bytes + text := fmt.Sprintf(`Compaction: compress message %d into one line of at most %d bytes (about 70 words), the length of this ruler: %s %s -`, task.Index, nodeBytes, compactorRuler, task.Input[0]), nil +`, task.Index, nodeBytes, compactorRuler, task.Input[0]) + if len(task.Images) > 0 { + text += "\nKeep each [image FILE] name and describe its kind, visible text, amounts, names, and context." + } + + return text, nil } if task.Level > 0 && len(task.Input) == 2 { diff --git a/compactor_test.go b/compactor_test.go index 8540515..baa7f57 100644 --- a/compactor_test.go +++ b/compactor_test.go @@ -208,7 +208,7 @@ func newTestCompactor(t *testing.T, server *ModelFake) *Compactor { return newCompactor(client, compactorAdaPrompt, ModelConfig{ Model: "gpt-test", Effort: "high", - }, promptTools()) + }, promptTools(), &ImageStore{dir: t.TempDir()}) } // compactorInputTexts returns the text of each user message of call. Each diff --git a/daemon_agent.go b/daemon_agent.go index 662291c..227d23b 100644 --- a/daemon_agent.go +++ b/daemon_agent.go @@ -55,7 +55,7 @@ func openDaemonAgents(ctx context.Context, dir, address string, auth ModelAuthor // one agent. All model calls use its cache key and usage file. func openDaemonAgent(ctx context.Context, agent *Agent, address string, auth ModelAuthorizer, compactor ModelConfig, clock func() time.Time, logger *slog.Logger) (*DaemonAgent, error) { client := newModelClient(address, auth, agent.CacheKey(), filepath.Join(agent.dir, "usage.jsonl")) - memory, err := openMemory(ctx, agent.dir, clock, newCompactor(client, agent.Prompt, compactor, agent.Tools()), logger) + memory, err := openMemory(ctx, agent.dir, clock, newCompactor(client, agent.Prompt, compactor, agent.Tools(), agent.Images()), logger) if err != nil { return nil, fmt.Errorf("open memory: %w", err) } diff --git a/image_model.go b/image_model.go new file mode 100644 index 0000000..6965142 --- /dev/null +++ b/image_model.go @@ -0,0 +1,90 @@ +package main + +import ( + "encoding/base64" + "errors" + "fmt" + "strings" +) + +// imageUserMessage attaches pixels only to messages selected as new by a turn. +func imageUserMessage(store *ImageStore, text string) (any, error) { + names := imageFiles(text) + if len(names) == 0 { + return modelUserMessage(text), nil + } + + parts, err := imageParts(store, text, names) + if err != nil { + return nil, fmt.Errorf("make image message: %w", err) + } + + return modelUserParts(parts), nil +} + +// imageParts is the shared wire format for image messages and zoom results. +// Tool results use this list directly as function_call_output.output. +func imageParts(store *ImageStore, text string, names []string) ([]ModelInputPart, error) { + parts := []ModelInputPart{{ + Type: "input_text", + Text: text, + }} + for _, name := range names { + if store == nil { + return nil, errors.New("image store is not available") + } + + data, err := store.Read(name) + if err != nil { + return nil, fmt.Errorf("read image part: %w", err) + } + + mime := "image/" + strings.TrimPrefix(name[64:], ".") + if strings.HasSuffix(name, ".jpg") { + mime = "image/jpeg" + } + + parts = append(parts, ModelInputPart{ + Type: "input_image", + ImageURL: "data:" + mime + ";base64," + base64.StdEncoding.EncodeToString(data), + }) + } + + return parts, nil +} + +// imageToolOutput leaves all results as text except zoom of one image message. +// Only the text goes to the log, never the data URLs. +func imageToolOutput(box *Toolbox, name, arguments, text string) (any, error) { + if name != "zoom" || strings.HasPrefix(text, "error:") { + return text, nil + } + + var args ToolZoomArguments + + err := toolDecodeArguments(arguments, &args) + if err != nil { + return nil, fmt.Errorf("decode image zoom: %w", err) + } + + if args.Name != nil || args.N != 1 { + return text, nil + } + + message, err := box.chat.Message(args.ID) + if err != nil { + return nil, fmt.Errorf("read image zoom: %w", err) + } + + names := imageFiles(message.Text) + if len(names) == 0 { + return text, nil + } + + parts, err := imageParts(box.images, text, names) + if err != nil { + return nil, fmt.Errorf("make image zoom: %w", err) + } + + return parts, nil +} diff --git a/image_model_test.go b/image_model_test.go new file mode 100644 index 0000000..53b0f31 --- /dev/null +++ b/image_model_test.go @@ -0,0 +1,199 @@ +package main + +import ( + "encoding/base64" + "encoding/json" + "net/http" + "path/filepath" + "strings" + "testing" + + "github.com/stretchr/testify/require" +) + +func TestNewTurnKeepsImagesInEveryChainRequestButNotLaterTurns(t *testing.T) { + server := newModelFake(t) + server.reply = func(w http.ResponseWriter, call ModelFakeCall) { + if call.number == 0 { + modelWriteEvents(w, modelCompletedEvent([]any{modelFunctionCallItem("fc_1", "call_1", "date", `{"id":0}`)}, 100, 0, 5)) + + return + } + + modelWriteEvents(w, modelCompletedEvent([]any{modelMessageItem("A red square.")}, 100, 0, 5)) + } + + test := newTestTurnLoop(t, server) + data := imageTestPNG(t, 3, 3) + name, err := test.agent.Images().Save(data) + require.NoError(t, err) + + marker := imageText([]string{name}, "What is this?") + _, err = test.loop.Submit(terminalSource, marker) + require.NoError(t, err) + test.events.WaitFor(t, "idle", 1) + require.Equal(t, 2, server.Calls()) + + first := server.Call(0).body["input"].([]any) + second := server.Call(1).body["input"].([]any) + require.Equal(t, first, second[:len(first)]) + + expected := "data:image/png;base64," + base64.StdEncoding.EncodeToString(data) + require.Contains(t, string(server.Call(0).raw), expected) + require.Contains(t, string(server.Call(1).raw), expected) + + _, err = test.loop.Submit(terminalSource, "Continue with text.") + require.NoError(t, err) + test.events.WaitFor(t, "idle", 2) + require.NotContains(t, string(server.Call(2).raw), "input_image") + require.NotContains(t, strings.Join(test.Log(t), "\n"), "base64") +} + +func TestImageArrivingDuringAChainJoinsItsNextRequest(t *testing.T) { + server := newModelFake(t) + arrived := make(chan struct{}) + release := make(chan struct{}) + server.reply = func(w http.ResponseWriter, call ModelFakeCall) { + if call.number == 0 { + close(arrived) + <-release + modelWriteEvents(w, modelCompletedEvent([]any{modelFunctionCallItem("fc_1", "call_1", "date", `{"id":0}`)}, 100, 0, 5)) + + return + } + + modelWriteEvents(w, modelCompletedEvent([]any{modelMessageItem("done")}, 100, 0, 5)) + } + + test := newTestTurnLoop(t, server) + _, err := test.loop.Submit(terminalSource, "Start.") + require.NoError(t, err) + <-arrived + + name, err := test.agent.Images().Save(imageTestPNG(t, 2, 2)) + require.NoError(t, err) + + _, err = test.loop.Submit(terminalSource, imageText([]string{name}, "Read this.")) + require.NoError(t, err) + close(release) + test.events.WaitFor(t, "idle", 1) + require.NotContains(t, string(server.Call(0).raw), "input_image") + require.Contains(t, string(server.Call(1).raw), "input_image") +} + +func TestLeafCompactionDescribesImagesOnceAndMergesOnlyText(t *testing.T) { + server := newModelFake(t) + store := &ImageStore{dir: filepath.Join(t.TempDir(), imageDirName)} + name, err := store.Save(imageTestPNG(t, 2, 2)) + require.NoError(t, err) + + server.reply = func(w http.ResponseWriter, call ModelFakeCall) { + text := "receipt, Ada, $12, red square" + if call.number == 0 { + text = strings.Repeat("detail ", 100) + } + + modelWriteEvents(w, modelCompletedEvent([]any{modelMessageItem(text)}, 100, 0, 5)) + } + + compactor := newTestCompactor(t, server) + compactor.images = store + task := Task{ + Level: 0, + Index: 0, + Context: "\n", + Input: []string{"user: " + imageText([]string{name}, "Read the receipt.")}, + Images: []string{name}, + } + text, err := compactor.Summarize(t.Context(), task) + require.NoError(t, err) + require.Equal(t, "receipt, Ada, $12, red square", text) + require.Contains(t, string(server.Call(0).raw), "input_image") + require.Contains(t, string(server.Call(0).raw), "kind, visible text, amounts, names, and context") + require.NotContains(t, string(server.Call(1).raw), "input_image") + + _, err = compactor.Summarize(t.Context(), Task{ + Level: 1, + Index: 0, + Context: "\n", + Input: []string{text, "user: next"}, + }) + require.NoError(t, err) + require.NotContains(t, string(server.Call(2).raw), "input_image") + + memory, err := openMemory(t.Context(), t.TempDir(), newTestClock().Now, compactor, discardLog()) + require.NoError(t, err) + t.Cleanup(func() { + require.NoError(t, memory.Close()) + }) + + _, err = memory.Append("user", imageText([]string{name}, "A short caption.")) + require.NoError(t, err) + require.NoError(t, memory.Settle(t.Context())) + require.Equal(t, 4, server.Calls()) + require.Contains(t, string(server.Call(3).raw), "input_image") + require.Contains(t, memory.View(), "receipt, Ada, $12") +} + +func TestZoomReturnsImagePartsButLogsOnlyText(t *testing.T) { + server := newModelFake(t) + server.reply = func(w http.ResponseWriter, call ModelFakeCall) { + if call.number == 0 { + modelWriteEvents(w, modelCompletedEvent([]any{modelFunctionCallItem("fc_1", "call_1", "zoom", `{"id":0,"n":1,"name":null}`)}, 100, 0, 5)) + + return + } + + modelWriteEvents(w, modelCompletedEvent([]any{modelMessageItem("A red square.")}, 100, 0, 5)) + } + + test := newTestTurnLoop(t, server) + name, err := test.agent.Images().Save(imageTestPNG(t, 2, 2)) + require.NoError(t, err) + + _, err = test.memory.Append("user", imageText([]string{name}, "Old image.")) + require.NoError(t, err) + + _, err = test.loop.Submit(terminalSource, "Zoom the old image.") + require.NoError(t, err) + test.events.WaitFor(t, "idle", 1) + require.NotContains(t, string(server.Call(0).raw), "input_image") + + input := server.Call(1).body["input"].([]any) + result := input[len(input)-1].(map[string]any) + require.Equal(t, "function_call_output", result["type"]) + + parts := result["output"].([]any) + require.Len(t, parts, 2) + require.Equal(t, "input_text", parts[0].(map[string]any)["type"]) + require.Equal(t, "input_image", parts[1].(map[string]any)["type"]) + require.Contains(t, parts[0].(map[string]any)["text"], "Old image.") + require.NotContains(t, strings.Join(test.Log(t), "\n"), "base64") +} + +func TestImageUserMessageKeepsTextPartsAndMarkerOrder(t *testing.T) { + store := &ImageStore{dir: t.TempDir()} + first, err := store.Save(imageTestPNG(t, 1, 2)) + require.NoError(t, err) + + second, err := store.Save(imageTestPNG(t, 2, 1)) + require.NoError(t, err) + + text := imageText([]string{first, second}, "Compare.") + item, err := imageUserMessage(store, text) + require.NoError(t, err) + + data, err := json.Marshal(item) + require.NoError(t, err) + + var message struct { + Role string `json:"role"` + Content []ModelInputPart `json:"content"` + } + + require.NoError(t, json.Unmarshal(data, &message)) + require.Equal(t, "user", message.Role) + require.Len(t, message.Content, 3) + require.Equal(t, text, message.Content[0].Text) + require.NotEqual(t, message.Content[1].ImageURL, message.Content[2].ImageURL) +} diff --git a/memory.go b/memory.go index a240b6c..602b86c 100644 --- a/memory.go +++ b/memory.go @@ -607,6 +607,7 @@ func (m *Memory) task(key NodeKey) (Task, error) { } task.Input = []string{leafSource(message)} + task.Images = imageFiles(message.Text) return task, nil } @@ -781,12 +782,14 @@ func (m *Memory) buildBuiltByModel(key NodeKey) (bool, error) { // Task is the build of the node (Level, Index). Context is the compaction view // for the build. Input holds the source "kind: text" of a leaf, or the two -// child lines of a merge with their newlines as spaces. +// child lines of a merge with their newlines as spaces. Images holds the leaf +// image filenames and is empty for merges. type Task struct { Level int Index int Context string Input []string + Images []string } // MemoryViewFile is the content of view.json. diff --git a/model.go b/model.go index 507fbb1..6ddd300 100644 --- a/model.go +++ b/model.go @@ -73,22 +73,22 @@ func newModelClient(url string, auth ModelAuthorizer, cacheKey, usagePath string // modelUserMessage returns a user input item with one text part for each text. func modelUserMessage(texts ...string) any { - type part struct { - Type string `json:"type"` - Text string `json:"text"` - } - - content := make([]part, len(texts)) + content := make([]ModelInputPart, len(texts)) for n, text := range texts { - content[n] = part{ + content[n] = ModelInputPart{ Type: "input_text", Text: text, } } + return modelUserParts(content) +} + +// modelUserParts keeps text and image parts in one user message. +func modelUserParts(content []ModelInputPart) any { return struct { - Role string `json:"role"` - Content []part `json:"content"` + Role string `json:"role"` + Content []ModelInputPart `json:"content"` }{ Role: "user", Content: content, @@ -321,6 +321,13 @@ type ModelOutputItem struct { } `json:"content"` } +// ModelInputPart is a text or image part in a user message or tool result. +type ModelInputPart struct { + Type string `json:"type"` + Text string `json:"text,omitempty"` + ImageURL string `json:"image_url,omitempty"` +} + // ModelRequest is one model call. Input holds the input items in order, such as // modelUserMessage values or output items of an earlier response. Node names // the node that a compaction builds, or is nil. Turn names the request of a diff --git a/probe.go b/probe.go index b004829..384d0f8 100644 --- a/probe.go +++ b/probe.go @@ -103,7 +103,7 @@ func probe(ctx context.Context, args []string, out io.Writer) (err error) { client.LimitCalls(probeCallLimit) summarizer := ProbeSlotSummarizer{ - inner: newCompactor(client, probePrompt, config.Compactor, promptTools()), + inner: newCompactor(client, probePrompt, config.Compactor, promptTools(), &ImageStore{dir: filepath.Join(dir, imageDirName)}), slots: make(chan struct{}, builds), } diff --git a/prompt.go b/prompt.go index 36fda32..df6f17d 100644 --- a/prompt.go +++ b/prompt.go @@ -41,10 +41,10 @@ var promptEfforts = []string{"low", "medium", "high", "xhigh"} // one added paragraph on the user's instructions file. The reference names its // own agent, so promptText has the name of the agent and the kind reply in its // place. It describes only what OneChat can do, so it leaves out the spec text -// on images, computers, and computer tasks. Its paragraph on persistent mode -// tells the model that a final answer ends the turn. prompt adds the clause on -// the instructions of the user only when there are instructions. It holds no -// date and no state, so turns and compactions share its prompt cache. +// on computers and computer tasks. Its paragraph on persistent mode tells the +// model that a final answer ends the turn. prompt adds the clause on the +// instructions of the user only when there are instructions. It holds no date +// and no state, so turns and compactions share its prompt cache. const promptText = `You are {name}, an AI agent that works for one user in a single chat that never ends. Each call to you is a turn or a compaction: the view below is followed by the user's new message, or by a task starting "Compaction:". @@ -76,6 +76,9 @@ Tools: - zoom("Name") gives an agent's whole chat - date(id) gives the date and time of message id +An [image FILE] line names a stored image. New images include pixels for this +turn only. Use zoom(id, 1) to see an older image again. + # Turns Do the user's tasks yourself, with your tools. @@ -155,10 +158,10 @@ Set or remove a route only after the user agrees.` // promptSubagentText is the system prompt of a subagent, from the subagent // prompt of the reference design: its head, then the doc of the view, without -// the parts on devices and images. A subagent gets the view of the agent, then -// its whole log, so it goes on from its log after a tell or a restart. -// promptSubagent adds the clause on the instructions of the user only when -// there are instructions. +// the parts on devices. A subagent gets the view of the agent, then its whole +// log, so it goes on from its log after a tell or a restart. promptSubagent +// adds the clause on the instructions of the user only when there are +// instructions. const promptSubagentText = `You are a subagent of {name}, an AI agent that works for one user in a single chat that never ends. {name} gave you a task. Do it yourself, with your tools. @@ -189,7 +192,8 @@ was made from; zoom(id, 1) gives message id in full. Zoom whenever a summary only mentions something you need, such as what your last reply said, a decision, a past attempt or where a file is, before you act, guess or ask. date(id) gives the date and time of message id. zoom("Name") gives an agent's -whole chat.` +whole chat. An [image FILE] line names a stored image. Use zoom(id, 1) +to see its pixels.` // prompt returns the system prompt of the reference with one added paragraph on // the user's instructions file at instructionsPath. It uses the agent name and diff --git a/prompt_test.go b/prompt_test.go index c9cadc2..e9e87f3 100644 --- a/prompt_test.go +++ b/prompt_test.go @@ -18,6 +18,7 @@ func TestPromptNamesTheAgent(t *testing.T) { require.Contains(t, text, `- work: an agent's report, starting "[Name]"`) require.Contains(t, text, "- zoom(id, 1) gives message id whole\n- zoom(\"Name\") gives an agent's whole chat\n") require.NotContains(t, text, promptName) + require.Contains(t, text, "[image FILE]") // The old agent name is built from parts, so the source does not spell // it. @@ -26,7 +27,7 @@ func TestPromptNamesTheAgent(t *testing.T) { func TestPromptLeavesOutAbsentFeatures(t *testing.T) { text := prompt("Ada", "/AGENT/instructions.md", "") - for _, absent := range []string{"image", "computer", "device", "instructions at the end"} { + for _, absent := range []string{"computer", "device", "instructions at the end"} { require.NotContains(t, text, absent) } } @@ -50,7 +51,7 @@ func TestSubagentPromptNamesTheAgentAndEndsWithTheInstructions(t *testing.T) { require.True(t, strings.HasSuffix(text, "\n\n# The user's instructions\n\nI am Lena.")) require.NotContains(t, text, promptName) - for _, absent := range []string{"image", "computer", "device", "Compaction"} { + for _, absent := range []string{"computer", "device", "Compaction"} { require.NotContains(t, text, absent) } diff --git a/rebuild_test.go b/rebuild_test.go index fc4ff7e..104ec88 100644 --- a/rebuild_test.go +++ b/rebuild_test.go @@ -63,7 +63,7 @@ func TestRebuildMergesDatesAndKeepsAgentFiles(t *testing.T) { }, } - backup, err := rebuild(t.Context(), agent, added, client, newCompactor(client, agent.Prompt, defaultConfig().Compactor, agent.Tools()), &report) + backup, err := rebuild(t.Context(), agent, added, client, newCompactor(client, agent.Prompt, defaultConfig().Compactor, agent.Tools(), agent.Images()), &report) require.NoError(t, err) require.Equal(t, before, rebuildTestSnapshot(t, backup)) @@ -143,7 +143,7 @@ func TestRebuildFailureAndCancellationKeepOriginal(t *testing.T) { Kind: "work", Text: strings.Repeat("long ", 200), Date: "2025-01-01T00:00:00Z", - }}, client, newCompactor(client, agent.Prompt, defaultConfig().Compactor, agent.Tools()), &strings.Builder{}) + }}, client, newCompactor(client, agent.Prompt, defaultConfig().Compactor, agent.Tools(), agent.Images()), &strings.Builder{}) require.Error(t, err) require.Empty(t, backup) require.Equal(t, before, rebuildTestSnapshot(t, agent.dir)) @@ -179,7 +179,7 @@ func TestRebuildKeepsOriginalWhenAnInvisibleMergeFails(t *testing.T) { } client := newModelClient(server.URL, ModelAPIKey(modelTestKey), agent.CacheKey(), filepath.Join(t.TempDir(), "usage.jsonl")) - _, err := rebuild(t.Context(), agent, nil, client, newCompactor(client, agent.Prompt, defaultConfig().Compactor, agent.Tools()), &strings.Builder{}) + _, err := rebuild(t.Context(), agent, nil, client, newCompactor(client, agent.Prompt, defaultConfig().Compactor, agent.Tools(), agent.Images()), &strings.Builder{}) require.ErrorContains(t, err, "summary tree has unfinished nodes") require.Equal(t, before, rebuildTestSnapshot(t, agent.dir)) } diff --git a/subagent_run.go b/subagent_run.go index 8084c39..891766c 100644 --- a/subagent_run.go +++ b/subagent_run.go @@ -515,11 +515,13 @@ func (r *SubagentRuns) chain(ctx context.Context, run *SubagentRun) (string, err input = append(input, modelUserMessage(r.state()), modelUserMessage(log)) chain := &Chain{ client: r.client, + images: r.agent.Images(), tools: &Toolbox{ log: run.subagent, chat: r.memory, subagents: r.subagents, workdir: run.subagent.WorkDir(), + images: r.agent.Images(), }, request: ModelRequest{ Model: r.agent.config.Model, diff --git a/tools.go b/tools.go index a75514f..96cbb97 100644 --- a/tools.go +++ b/tools.go @@ -68,13 +68,15 @@ type Toolbox struct { messages *DaemonMessages children *DaemonChildren routes *Routes + images *ImageStore } // Call logs the call of the tool name with its JSON arguments, makes the log -// durable, runs the tool, and logs its clipped output, which it returns. A -// failed tool gives its error as the output. An error of Call itself is an -// error of the log. When ctx ends before the flush, the tool does not run. -func (b *Toolbox) Call(ctx context.Context, name, arguments string) (string, error) { +// durable, runs the tool, and logs its clipped text. It returns that text or +// the text and image parts of a single-message zoom. Pixels never enter the +// log. A failed tool gives its error as the output. An error of Call itself is +// an error of the log. When ctx ends before the flush, the tool does not run. +func (b *Toolbox) Call(ctx context.Context, name, arguments string) (any, error) { _, err := b.log.Append("tool", toolCallText(name, arguments)) if err != nil { return "", fmt.Errorf("append tool call: %w", err) @@ -90,12 +92,26 @@ func (b *Toolbox) Call(ctx context.Context, name, arguments string) (string, err output = toolClip(b.run(ctx, name, arguments)) } + var ( + result any = output + resultErr error + ) + + if err == nil { + result, resultErr = imageToolOutput(b, name, arguments, output) + } + + if resultErr != nil { + output = "error: " + resultErr.Error() + result = output + } + _, err = b.log.Append("echo", output) if err != nil { return "", fmt.Errorf("append tool result: %w", err) } - return output, nil + return result, nil } // run runs one tool and returns its output, or its error as text. diff --git a/tree.go b/tree.go index 69558c5..079969c 100644 --- a/tree.go +++ b/tree.go @@ -55,11 +55,11 @@ func leafSource(message Message) string { return message.Kind + ": " + message.Text } -// freeLeaf returns the leaf of message when its source fits in a node. Such a -// leaf holds the source word for word and needs no model call. +// freeLeaf returns a text-only leaf when its source fits in a node. Images need +// a model description even when their text fits. func freeLeaf(message Message) (Node, bool) { text := leafSource(message) - if len(text) > nodeBytes { + if len(text) > nodeBytes || len(imageFiles(message.Text)) > 0 { return Node{}, false } diff --git a/turn.go b/turn.go index e0bd489..2c1d9c2 100644 --- a/turn.go +++ b/turn.go @@ -60,6 +60,7 @@ func newTurnLoop(agent *Agent, memory *Memory, subagents *Subagents, whatsapp *W schedules: schedules, workdir: agent.config.Workdir, agentName: agent.config.Name, + images: agent.Images(), } return l, nil @@ -423,10 +424,19 @@ func (l *TurnLoop) chain(ctx context.Context) (string, error) { Instructions: instructions, Tools: l.agent.Tools(), }, - take: l.take, + take: l.take, + images: l.agent.Images(), } - reply, err := chain.Run(ctx, turnInput(view, l.state(), messages)) + input := turnInput(view, l.state(), messages) + for n, message := range messages { + input[len(input)-len(messages)+n], err = imageUserMessage(l.agent.Images(), message.Text) + if err != nil { + return "", fmt.Errorf("read turn images: %w", err) + } + } + + reply, err := chain.Run(ctx, input) if err != nil { return "", fmt.Errorf("run turn chain: %w", err) } diff --git a/whatsapp_import.go b/whatsapp_import.go index df871c1..1092b49 100644 --- a/whatsapp_import.go +++ b/whatsapp_import.go @@ -143,7 +143,7 @@ func whatsappImport(ctx context.Context, args []string, out io.Writer) (err erro } client := newModelClient(address, auth, agent.CacheKey(), usagePath) - backup, err := rebuild(ctx, agent, plan.added, client, newCompactor(client, agent.Prompt, config.Compactor, agent.Tools()), out) + backup, err := rebuild(ctx, agent, plan.added, client, newCompactor(client, agent.Prompt, config.Compactor, agent.Tools(), agent.Images()), out) if err != nil { return fmt.Errorf("rebuild imported memory: %w", err) } diff --git a/whatsapp_import_test.go b/whatsapp_import_test.go index 1f9c9af..4bd0a1f 100644 --- a/whatsapp_import_test.go +++ b/whatsapp_import_test.go @@ -563,7 +563,7 @@ func TestWhatsAppImportChildRebuildUsesOwnPromptAndCache(t *testing.T) { } client := newModelClient(server.URL, ModelAPIKey(modelTestKey), child.CacheKey(), filepath.Join(t.TempDir(), "usage.jsonl")) - backup, err := rebuild(t.Context(), child, plan.added, client, newCompactor(client, child.Prompt, defaultConfig().Compactor, child.Tools()), &strings.Builder{}) + backup, err := rebuild(t.Context(), child, plan.added, client, newCompactor(client, child.Prompt, defaultConfig().Compactor, child.Tools(), child.Images()), &strings.Builder{}) require.NoError(t, err) require.Equal(t, before, rebuildTestSnapshot(t, backup)) require.Equal(t, root.dir, filepath.Dir(backup)) @@ -613,7 +613,7 @@ func TestWhatsAppImportChildFailureKeepsConfiguredChildren(t *testing.T) { Kind: "work", Text: strings.Repeat("The plan is ready. ", 80), Date: "2025-01-01T08:00:00Z", - }}, client, newCompactor(client, child.Prompt, defaultConfig().Compactor, child.Tools()), &strings.Builder{}) + }}, client, newCompactor(client, child.Prompt, defaultConfig().Compactor, child.Tools(), child.Images()), &strings.Builder{}) require.ErrorContains(t, err, "rebuild") require.Equal(t, before, rebuildTestSnapshot(t, root.dir))