Attach stored pixels to new user messages for the whole turn chain.
Later turns keep text markers and descriptions, which keeps the request
prefix stable. Leaf compaction sees pixels once per build and preserves
visible details. Single-message zoom returns text and image parts while
the durable log keeps only text.
root := t.TempDir()
usagePath := filepath.Join(root, "usage.jsonl")
client := newModelClient(server.URL, ModelAPIKey(modelTestKey), "onechat-"+probeName, usagePath)
- memory, err := openMemory(t.Context(), root, newTestClock().Now, newCompactor(client, probePrompt, defaultConfig().Compactor, promptTools()), discardLog())
+ memory, err := openMemory(t.Context(), root, newTestClock().Now, newCompactor(client, probePrompt, defaultConfig().Compactor, promptTools(), &ImageStore{dir: t.TempDir()}), discardLog())
require.NoError(t, err)
return &BuildRun{
tools *Toolbox
request ModelRequest
take func() []Message
+ images *ImageStore
}
// Run runs the chain from input, the input of its first request, and returns
input = append(input, outputs...)
for _, message := range c.take() {
- input = append(input, modelUserMessage(message.Text))
+ item, err := imageUserMessage(c.images, message.Text)
+ if err != nil {
+ return nil, "", fmt.Errorf("read queued images: %w", err)
+ }
+
+ input = append(input, item)
}
return input, "", nil
type ChainFunctionCallOutput struct {
Type string `json:"type"`
CallID string `json:"call_id"`
- Output string `json:"output"`
+ Output any `json:"output"`
}
// chainInputItem returns an output item as an input item for the next request.
"context"
"errors"
"fmt"
- "slices"
"strings"
"unicode/utf8"
)
// client with model. instructions gives the system prompt of the agent at each
// build. tools are the tools of that agent, so a build sends the same prompt
// and tools as its turns.
-func newCompactor(client *ModelClient, instructions func() (string, error), model ModelConfig, tools []ModelTool) *Compactor {
+func newCompactor(client *ModelClient, instructions func() (string, error), model ModelConfig, tools []ModelTool, images *ImageStore) *Compactor {
return &Compactor{
client: client,
model: model,
instructions: instructions,
tools: tools,
+ images: images,
}
}
//
// When the line is over nodeBytes, the next try sends the same message, plus
// one more user message with a note with the size of the newest line and its
-// cut at the limit. A try never sends the model's earlier reply or reasoning
-// back, so every try begins with the same view and task and has about the same
-// size. After at most compactorTries tries, the shortest line is the node. A
-// few bytes over the limit is fine, because the view measures the real sizes. A
-// call that fails before any line fails the build, and the core tries it again
-// at the next message.
+// cut at the limit. Only the first call of an image leaf sends its pixels. A
+// try never sends the model's earlier reply or reasoning back, so every try
+// begins with the same view and task and has about the same size. After at most
+// compactorTries tries, the shortest line is the node. A few bytes over the
+// limit is fine, because the view measures the real sizes. A call that fails
+// before any line fails the build, and the core tries it again at the next
+// message.
type Compactor struct {
client *ModelClient
model ModelConfig
instructions func() (string, error)
tools []ModelTool
+ images *ImageStore
}
// Summarize returns the line of the node of task.
return "", err
}
- first := []any{modelUserMessage(task.Context + "\n" + text)}
+ source := task.Context + "\n" + text
+ parts, err := imageParts(c.images, source, task.Images)
+ if err != nil {
+ return "", fmt.Errorf("read compaction images: %w", err)
+ }
+
+ first := []any{modelUserParts(parts)}
var lines []string
for len(lines) < compactorTries {
input := first
if note != "" {
- input = append(slices.Clip(first), modelUserMessage(note))
+ input = []any{modelUserMessage(source), modelUserMessage(note)}
}
var response ModelResponse
// compactorTask returns the task of a build, with its input.
func compactorTask(task Task) (string, error) {
if task.Level == 0 && len(task.Input) == 1 {
- return fmt.Sprintf(`Compaction: compress message %d into one line of at most %d bytes
+ text := fmt.Sprintf(`Compaction: compress message %d into one line of at most %d bytes
(about 70 words), the length of this ruler:
%s
<input>
%s
-</input>`, task.Index, nodeBytes, compactorRuler, task.Input[0]), nil
+</input>`, task.Index, nodeBytes, compactorRuler, task.Input[0])
+ if len(task.Images) > 0 {
+ text += "\nKeep each [image FILE] name and describe its kind, visible text, amounts, names, and context."
+ }
+
+ return text, nil
}
if task.Level > 0 && len(task.Input) == 2 {
return newCompactor(client, compactorAdaPrompt, ModelConfig{
Model: "gpt-test",
Effort: "high",
- }, promptTools())
+ }, promptTools(), &ImageStore{dir: t.TempDir()})
}
// compactorInputTexts returns the text of each user message of call. Each
// one agent. All model calls use its cache key and usage file.
func openDaemonAgent(ctx context.Context, agent *Agent, address string, auth ModelAuthorizer, compactor ModelConfig, clock func() time.Time, logger *slog.Logger) (*DaemonAgent, error) {
client := newModelClient(address, auth, agent.CacheKey(), filepath.Join(agent.dir, "usage.jsonl"))
- memory, err := openMemory(ctx, agent.dir, clock, newCompactor(client, agent.Prompt, compactor, agent.Tools()), logger)
+ memory, err := openMemory(ctx, agent.dir, clock, newCompactor(client, agent.Prompt, compactor, agent.Tools(), agent.Images()), logger)
if err != nil {
return nil, fmt.Errorf("open memory: %w", err)
}
--- /dev/null
+package main
+
+import (
+ "encoding/base64"
+ "errors"
+ "fmt"
+ "strings"
+)
+
+// imageUserMessage attaches pixels only to messages selected as new by a turn.
+func imageUserMessage(store *ImageStore, text string) (any, error) {
+ names := imageFiles(text)
+ if len(names) == 0 {
+ return modelUserMessage(text), nil
+ }
+
+ parts, err := imageParts(store, text, names)
+ if err != nil {
+ return nil, fmt.Errorf("make image message: %w", err)
+ }
+
+ return modelUserParts(parts), nil
+}
+
+// imageParts is the shared wire format for image messages and zoom results.
+// Tool results use this list directly as function_call_output.output.
+func imageParts(store *ImageStore, text string, names []string) ([]ModelInputPart, error) {
+ parts := []ModelInputPart{{
+ Type: "input_text",
+ Text: text,
+ }}
+ for _, name := range names {
+ if store == nil {
+ return nil, errors.New("image store is not available")
+ }
+
+ data, err := store.Read(name)
+ if err != nil {
+ return nil, fmt.Errorf("read image part: %w", err)
+ }
+
+ mime := "image/" + strings.TrimPrefix(name[64:], ".")
+ if strings.HasSuffix(name, ".jpg") {
+ mime = "image/jpeg"
+ }
+
+ parts = append(parts, ModelInputPart{
+ Type: "input_image",
+ ImageURL: "data:" + mime + ";base64," + base64.StdEncoding.EncodeToString(data),
+ })
+ }
+
+ return parts, nil
+}
+
+// imageToolOutput leaves all results as text except zoom of one image message.
+// Only the text goes to the log, never the data URLs.
+func imageToolOutput(box *Toolbox, name, arguments, text string) (any, error) {
+ if name != "zoom" || strings.HasPrefix(text, "error:") {
+ return text, nil
+ }
+
+ var args ToolZoomArguments
+
+ err := toolDecodeArguments(arguments, &args)
+ if err != nil {
+ return nil, fmt.Errorf("decode image zoom: %w", err)
+ }
+
+ if args.Name != nil || args.N != 1 {
+ return text, nil
+ }
+
+ message, err := box.chat.Message(args.ID)
+ if err != nil {
+ return nil, fmt.Errorf("read image zoom: %w", err)
+ }
+
+ names := imageFiles(message.Text)
+ if len(names) == 0 {
+ return text, nil
+ }
+
+ parts, err := imageParts(box.images, text, names)
+ if err != nil {
+ return nil, fmt.Errorf("make image zoom: %w", err)
+ }
+
+ return parts, nil
+}
--- /dev/null
+package main
+
+import (
+ "encoding/base64"
+ "encoding/json"
+ "net/http"
+ "path/filepath"
+ "strings"
+ "testing"
+
+ "github.com/stretchr/testify/require"
+)
+
+func TestNewTurnKeepsImagesInEveryChainRequestButNotLaterTurns(t *testing.T) {
+ server := newModelFake(t)
+ server.reply = func(w http.ResponseWriter, call ModelFakeCall) {
+ if call.number == 0 {
+ modelWriteEvents(w, modelCompletedEvent([]any{modelFunctionCallItem("fc_1", "call_1", "date", `{"id":0}`)}, 100, 0, 5))
+
+ return
+ }
+
+ modelWriteEvents(w, modelCompletedEvent([]any{modelMessageItem("A red square.")}, 100, 0, 5))
+ }
+
+ test := newTestTurnLoop(t, server)
+ data := imageTestPNG(t, 3, 3)
+ name, err := test.agent.Images().Save(data)
+ require.NoError(t, err)
+
+ marker := imageText([]string{name}, "What is this?")
+ _, err = test.loop.Submit(terminalSource, marker)
+ require.NoError(t, err)
+ test.events.WaitFor(t, "idle", 1)
+ require.Equal(t, 2, server.Calls())
+
+ first := server.Call(0).body["input"].([]any)
+ second := server.Call(1).body["input"].([]any)
+ require.Equal(t, first, second[:len(first)])
+
+ expected := "data:image/png;base64," + base64.StdEncoding.EncodeToString(data)
+ require.Contains(t, string(server.Call(0).raw), expected)
+ require.Contains(t, string(server.Call(1).raw), expected)
+
+ _, err = test.loop.Submit(terminalSource, "Continue with text.")
+ require.NoError(t, err)
+ test.events.WaitFor(t, "idle", 2)
+ require.NotContains(t, string(server.Call(2).raw), "input_image")
+ require.NotContains(t, strings.Join(test.Log(t), "\n"), "base64")
+}
+
+func TestImageArrivingDuringAChainJoinsItsNextRequest(t *testing.T) {
+ server := newModelFake(t)
+ arrived := make(chan struct{})
+ release := make(chan struct{})
+ server.reply = func(w http.ResponseWriter, call ModelFakeCall) {
+ if call.number == 0 {
+ close(arrived)
+ <-release
+ modelWriteEvents(w, modelCompletedEvent([]any{modelFunctionCallItem("fc_1", "call_1", "date", `{"id":0}`)}, 100, 0, 5))
+
+ return
+ }
+
+ modelWriteEvents(w, modelCompletedEvent([]any{modelMessageItem("done")}, 100, 0, 5))
+ }
+
+ test := newTestTurnLoop(t, server)
+ _, err := test.loop.Submit(terminalSource, "Start.")
+ require.NoError(t, err)
+ <-arrived
+
+ name, err := test.agent.Images().Save(imageTestPNG(t, 2, 2))
+ require.NoError(t, err)
+
+ _, err = test.loop.Submit(terminalSource, imageText([]string{name}, "Read this."))
+ require.NoError(t, err)
+ close(release)
+ test.events.WaitFor(t, "idle", 1)
+ require.NotContains(t, string(server.Call(0).raw), "input_image")
+ require.Contains(t, string(server.Call(1).raw), "input_image")
+}
+
+func TestLeafCompactionDescribesImagesOnceAndMergesOnlyText(t *testing.T) {
+ server := newModelFake(t)
+ store := &ImageStore{dir: filepath.Join(t.TempDir(), imageDirName)}
+ name, err := store.Save(imageTestPNG(t, 2, 2))
+ require.NoError(t, err)
+
+ server.reply = func(w http.ResponseWriter, call ModelFakeCall) {
+ text := "receipt, Ada, $12, red square"
+ if call.number == 0 {
+ text = strings.Repeat("detail ", 100)
+ }
+
+ modelWriteEvents(w, modelCompletedEvent([]any{modelMessageItem(text)}, 100, 0, 5))
+ }
+
+ compactor := newTestCompactor(t, server)
+ compactor.images = store
+ task := Task{
+ Level: 0,
+ Index: 0,
+ Context: "<chat>\n</chat>",
+ Input: []string{"user: " + imageText([]string{name}, "Read the receipt.")},
+ Images: []string{name},
+ }
+ text, err := compactor.Summarize(t.Context(), task)
+ require.NoError(t, err)
+ require.Equal(t, "receipt, Ada, $12, red square", text)
+ require.Contains(t, string(server.Call(0).raw), "input_image")
+ require.Contains(t, string(server.Call(0).raw), "kind, visible text, amounts, names, and context")
+ require.NotContains(t, string(server.Call(1).raw), "input_image")
+
+ _, err = compactor.Summarize(t.Context(), Task{
+ Level: 1,
+ Index: 0,
+ Context: "<chat>\n</chat>",
+ Input: []string{text, "user: next"},
+ })
+ require.NoError(t, err)
+ require.NotContains(t, string(server.Call(2).raw), "input_image")
+
+ memory, err := openMemory(t.Context(), t.TempDir(), newTestClock().Now, compactor, discardLog())
+ require.NoError(t, err)
+ t.Cleanup(func() {
+ require.NoError(t, memory.Close())
+ })
+
+ _, err = memory.Append("user", imageText([]string{name}, "A short caption."))
+ require.NoError(t, err)
+ require.NoError(t, memory.Settle(t.Context()))
+ require.Equal(t, 4, server.Calls())
+ require.Contains(t, string(server.Call(3).raw), "input_image")
+ require.Contains(t, memory.View(), "receipt, Ada, $12")
+}
+
+func TestZoomReturnsImagePartsButLogsOnlyText(t *testing.T) {
+ server := newModelFake(t)
+ server.reply = func(w http.ResponseWriter, call ModelFakeCall) {
+ if call.number == 0 {
+ modelWriteEvents(w, modelCompletedEvent([]any{modelFunctionCallItem("fc_1", "call_1", "zoom", `{"id":0,"n":1,"name":null}`)}, 100, 0, 5))
+
+ return
+ }
+
+ modelWriteEvents(w, modelCompletedEvent([]any{modelMessageItem("A red square.")}, 100, 0, 5))
+ }
+
+ test := newTestTurnLoop(t, server)
+ name, err := test.agent.Images().Save(imageTestPNG(t, 2, 2))
+ require.NoError(t, err)
+
+ _, err = test.memory.Append("user", imageText([]string{name}, "Old image."))
+ require.NoError(t, err)
+
+ _, err = test.loop.Submit(terminalSource, "Zoom the old image.")
+ require.NoError(t, err)
+ test.events.WaitFor(t, "idle", 1)
+ require.NotContains(t, string(server.Call(0).raw), "input_image")
+
+ input := server.Call(1).body["input"].([]any)
+ result := input[len(input)-1].(map[string]any)
+ require.Equal(t, "function_call_output", result["type"])
+
+ parts := result["output"].([]any)
+ require.Len(t, parts, 2)
+ require.Equal(t, "input_text", parts[0].(map[string]any)["type"])
+ require.Equal(t, "input_image", parts[1].(map[string]any)["type"])
+ require.Contains(t, parts[0].(map[string]any)["text"], "Old image.")
+ require.NotContains(t, strings.Join(test.Log(t), "\n"), "base64")
+}
+
+func TestImageUserMessageKeepsTextPartsAndMarkerOrder(t *testing.T) {
+ store := &ImageStore{dir: t.TempDir()}
+ first, err := store.Save(imageTestPNG(t, 1, 2))
+ require.NoError(t, err)
+
+ second, err := store.Save(imageTestPNG(t, 2, 1))
+ require.NoError(t, err)
+
+ text := imageText([]string{first, second}, "Compare.")
+ item, err := imageUserMessage(store, text)
+ require.NoError(t, err)
+
+ data, err := json.Marshal(item)
+ require.NoError(t, err)
+
+ var message struct {
+ Role string `json:"role"`
+ Content []ModelInputPart `json:"content"`
+ }
+
+ require.NoError(t, json.Unmarshal(data, &message))
+ require.Equal(t, "user", message.Role)
+ require.Len(t, message.Content, 3)
+ require.Equal(t, text, message.Content[0].Text)
+ require.NotEqual(t, message.Content[1].ImageURL, message.Content[2].ImageURL)
+}
}
task.Input = []string{leafSource(message)}
+ task.Images = imageFiles(message.Text)
return task, nil
}
// Task is the build of the node (Level, Index). Context is the compaction view
// for the build. Input holds the source "kind: text" of a leaf, or the two
-// child lines of a merge with their newlines as spaces.
+// child lines of a merge with their newlines as spaces. Images holds the leaf
+// image filenames and is empty for merges.
type Task struct {
Level int
Index int
Context string
Input []string
+ Images []string
}
// MemoryViewFile is the content of view.json.
// modelUserMessage returns a user input item with one text part for each text.
func modelUserMessage(texts ...string) any {
- type part struct {
- Type string `json:"type"`
- Text string `json:"text"`
- }
-
- content := make([]part, len(texts))
+ content := make([]ModelInputPart, len(texts))
for n, text := range texts {
- content[n] = part{
+ content[n] = ModelInputPart{
Type: "input_text",
Text: text,
}
}
+ return modelUserParts(content)
+}
+
+// modelUserParts keeps text and image parts in one user message.
+func modelUserParts(content []ModelInputPart) any {
return struct {
- Role string `json:"role"`
- Content []part `json:"content"`
+ Role string `json:"role"`
+ Content []ModelInputPart `json:"content"`
}{
Role: "user",
Content: content,
} `json:"content"`
}
+// ModelInputPart is a text or image part in a user message or tool result.
+type ModelInputPart struct {
+ Type string `json:"type"`
+ Text string `json:"text,omitempty"`
+ ImageURL string `json:"image_url,omitempty"`
+}
+
// ModelRequest is one model call. Input holds the input items in order, such as
// modelUserMessage values or output items of an earlier response. Node names
// the node that a compaction builds, or is nil. Turn names the request of a
client.LimitCalls(probeCallLimit)
summarizer := ProbeSlotSummarizer{
- inner: newCompactor(client, probePrompt, config.Compactor, promptTools()),
+ inner: newCompactor(client, probePrompt, config.Compactor, promptTools(), &ImageStore{dir: filepath.Join(dir, imageDirName)}),
slots: make(chan struct{}, builds),
}
// one added paragraph on the user's instructions file. The reference names its
// own agent, so promptText has the name of the agent and the kind reply in its
// place. It describes only what OneChat can do, so it leaves out the spec text
-// on images, computers, and computer tasks. Its paragraph on persistent mode
-// tells the model that a final answer ends the turn. prompt adds the clause on
-// the instructions of the user only when there are instructions. It holds no
-// date and no state, so turns and compactions share its prompt cache.
+// on computers and computer tasks. Its paragraph on persistent mode tells the
+// model that a final answer ends the turn. prompt adds the clause on the
+// instructions of the user only when there are instructions. It holds no date
+// and no state, so turns and compactions share its prompt cache.
const promptText = `You are {name}, an AI agent that works for one user in a single chat that never
ends. Each call to you is a turn or a compaction: the view below is followed by
the user's new message, or by a task starting "Compaction:".
- zoom("Name") gives an agent's whole chat
- date(id) gives the date and time of message id
+An [image FILE] line names a stored image. New images include pixels for this
+turn only. Use zoom(id, 1) to see an older image again.
+
# Turns
Do the user's tasks yourself, with your tools.
// promptSubagentText is the system prompt of a subagent, from the subagent
// prompt of the reference design: its head, then the doc of the view, without
-// the parts on devices and images. A subagent gets the view of the agent, then
-// its whole log, so it goes on from its log after a tell or a restart.
-// promptSubagent adds the clause on the instructions of the user only when
-// there are instructions.
+// the parts on devices. A subagent gets the view of the agent, then its whole
+// log, so it goes on from its log after a tell or a restart. promptSubagent
+// adds the clause on the instructions of the user only when there are
+// instructions.
const promptSubagentText = `You are a subagent of {name}, an AI agent that works for one user in a single
chat that never ends. {name} gave you a task. Do it yourself, with your tools.
only mentions something you need, such as what your last reply said, a
decision, a past attempt or where a file is, before you act, guess or ask.
date(id) gives the date and time of message id. zoom("Name") gives an agent's
-whole chat.`
+whole chat. An [image FILE] line names a stored image. Use zoom(id, 1)
+to see its pixels.`
// prompt returns the system prompt of the reference with one added paragraph on
// the user's instructions file at instructionsPath. It uses the agent name and
require.Contains(t, text, `- work: an agent's report, starting "[Name]"`)
require.Contains(t, text, "- zoom(id, 1) gives message id whole\n- zoom(\"Name\") gives an agent's whole chat\n")
require.NotContains(t, text, promptName)
+ require.Contains(t, text, "[image FILE]")
// The old agent name is built from parts, so the source does not spell
// it.
func TestPromptLeavesOutAbsentFeatures(t *testing.T) {
text := prompt("Ada", "/AGENT/instructions.md", "")
- for _, absent := range []string{"image", "computer", "device", "instructions at the end"} {
+ for _, absent := range []string{"computer", "device", "instructions at the end"} {
require.NotContains(t, text, absent)
}
}
require.True(t, strings.HasSuffix(text, "\n\n# The user's instructions\n\nI am Lena."))
require.NotContains(t, text, promptName)
- for _, absent := range []string{"image", "computer", "device", "Compaction"} {
+ for _, absent := range []string{"computer", "device", "Compaction"} {
require.NotContains(t, text, absent)
}
},
}
- backup, err := rebuild(t.Context(), agent, added, client, newCompactor(client, agent.Prompt, defaultConfig().Compactor, agent.Tools()), &report)
+ backup, err := rebuild(t.Context(), agent, added, client, newCompactor(client, agent.Prompt, defaultConfig().Compactor, agent.Tools(), agent.Images()), &report)
require.NoError(t, err)
require.Equal(t, before, rebuildTestSnapshot(t, backup))
Kind: "work",
Text: strings.Repeat("long ", 200),
Date: "2025-01-01T00:00:00Z",
- }}, client, newCompactor(client, agent.Prompt, defaultConfig().Compactor, agent.Tools()), &strings.Builder{})
+ }}, client, newCompactor(client, agent.Prompt, defaultConfig().Compactor, agent.Tools(), agent.Images()), &strings.Builder{})
require.Error(t, err)
require.Empty(t, backup)
require.Equal(t, before, rebuildTestSnapshot(t, agent.dir))
}
client := newModelClient(server.URL, ModelAPIKey(modelTestKey), agent.CacheKey(), filepath.Join(t.TempDir(), "usage.jsonl"))
- _, err := rebuild(t.Context(), agent, nil, client, newCompactor(client, agent.Prompt, defaultConfig().Compactor, agent.Tools()), &strings.Builder{})
+ _, err := rebuild(t.Context(), agent, nil, client, newCompactor(client, agent.Prompt, defaultConfig().Compactor, agent.Tools(), agent.Images()), &strings.Builder{})
require.ErrorContains(t, err, "summary tree has unfinished nodes")
require.Equal(t, before, rebuildTestSnapshot(t, agent.dir))
}
input = append(input, modelUserMessage(r.state()), modelUserMessage(log))
chain := &Chain{
client: r.client,
+ images: r.agent.Images(),
tools: &Toolbox{
log: run.subagent,
chat: r.memory,
subagents: r.subagents,
workdir: run.subagent.WorkDir(),
+ images: r.agent.Images(),
},
request: ModelRequest{
Model: r.agent.config.Model,
messages *DaemonMessages
children *DaemonChildren
routes *Routes
+ images *ImageStore
}
// Call logs the call of the tool name with its JSON arguments, makes the log
-// durable, runs the tool, and logs its clipped output, which it returns. A
-// failed tool gives its error as the output. An error of Call itself is an
-// error of the log. When ctx ends before the flush, the tool does not run.
-func (b *Toolbox) Call(ctx context.Context, name, arguments string) (string, error) {
+// durable, runs the tool, and logs its clipped text. It returns that text or
+// the text and image parts of a single-message zoom. Pixels never enter the
+// log. A failed tool gives its error as the output. An error of Call itself is
+// an error of the log. When ctx ends before the flush, the tool does not run.
+func (b *Toolbox) Call(ctx context.Context, name, arguments string) (any, error) {
_, err := b.log.Append("tool", toolCallText(name, arguments))
if err != nil {
return "", fmt.Errorf("append tool call: %w", err)
output = toolClip(b.run(ctx, name, arguments))
}
+ var (
+ result any = output
+ resultErr error
+ )
+
+ if err == nil {
+ result, resultErr = imageToolOutput(b, name, arguments, output)
+ }
+
+ if resultErr != nil {
+ output = "error: " + resultErr.Error()
+ result = output
+ }
+
_, err = b.log.Append("echo", output)
if err != nil {
return "", fmt.Errorf("append tool result: %w", err)
}
- return output, nil
+ return result, nil
}
// run runs one tool and returns its output, or its error as text.
return message.Kind + ": " + message.Text
}
-// freeLeaf returns the leaf of message when its source fits in a node. Such a
-// leaf holds the source word for word and needs no model call.
+// freeLeaf returns a text-only leaf when its source fits in a node. Images need
+// a model description even when their text fits.
func freeLeaf(message Message) (Node, bool) {
text := leafSource(message)
- if len(text) > nodeBytes {
+ if len(text) > nodeBytes || len(imageFiles(message.Text)) > 0 {
return Node{}, false
}
schedules: schedules,
workdir: agent.config.Workdir,
agentName: agent.config.Name,
+ images: agent.Images(),
}
return l, nil
Instructions: instructions,
Tools: l.agent.Tools(),
},
- take: l.take,
+ take: l.take,
+ images: l.agent.Images(),
}
- reply, err := chain.Run(ctx, turnInput(view, l.state(), messages))
+ input := turnInput(view, l.state(), messages)
+ for n, message := range messages {
+ input[len(input)-len(messages)+n], err = imageUserMessage(l.agent.Images(), message.Text)
+ if err != nil {
+ return "", fmt.Errorf("read turn images: %w", err)
+ }
+ }
+
+ reply, err := chain.Run(ctx, input)
if err != nil {
return "", fmt.Errorf("run turn chain: %w", err)
}
}
client := newModelClient(address, auth, agent.CacheKey(), usagePath)
- backup, err := rebuild(ctx, agent, plan.added, client, newCompactor(client, agent.Prompt, config.Compactor, agent.Tools()), out)
+ backup, err := rebuild(ctx, agent, plan.added, client, newCompactor(client, agent.Prompt, config.Compactor, agent.Tools(), agent.Images()), out)
if err != nil {
return fmt.Errorf("rebuild imported memory: %w", err)
}
}
client := newModelClient(server.URL, ModelAPIKey(modelTestKey), child.CacheKey(), filepath.Join(t.TempDir(), "usage.jsonl"))
- backup, err := rebuild(t.Context(), child, plan.added, client, newCompactor(client, child.Prompt, defaultConfig().Compactor, child.Tools()), &strings.Builder{})
+ backup, err := rebuild(t.Context(), child, plan.added, client, newCompactor(client, child.Prompt, defaultConfig().Compactor, child.Tools(), child.Images()), &strings.Builder{})
require.NoError(t, err)
require.Equal(t, before, rebuildTestSnapshot(t, backup))
require.Equal(t, root.dir, filepath.Dir(backup))
Kind: "work",
Text: strings.Repeat("The plan is ready. ", 80),
Date: "2025-01-01T08:00:00Z",
- }}, client, newCompactor(client, child.Prompt, defaultConfig().Compactor, child.Tools()), &strings.Builder{})
+ }}, client, newCompactor(client, child.Prompt, defaultConfig().Compactor, child.Tools(), child.Images()), &strings.Builder{})
require.ErrorContains(t, err, "rebuild")
require.Equal(t, before, rebuildTestSnapshot(t, root.dir))
projects / onechat / commitdiff