Carry new images through model request chains

Attach stored pixels to new user messages for the whole turn chain.
Later turns keep text markers and descriptions, which keeps the request
prefix stable. Leaf compaction sees pixels once per build and preserves
visible details. Single-message zoom returns text and image parts while
the durable log keeps only text.

diff --git a/build_run_test.go b/build_run_test.go
index b45745e7590b7ee8051c2f6f8a4df14208952a2e..ef9c3c0fb48c98582888f440854a32d5577e6463 100644 (file)
--- a/build_run_test.go
+++ b/build_run_test.go
@@ -215,7 +215,7 @@ func newTestBuildRun(t *testing.T, server *ModelFake) *BuildRun {
        root := t.TempDir()
        usagePath := filepath.Join(root, "usage.jsonl")
        client := newModelClient(server.URL, ModelAPIKey(modelTestKey), "onechat-"+probeName, usagePath)
        root := t.TempDir()
        usagePath := filepath.Join(root, "usage.jsonl")
        client := newModelClient(server.URL, ModelAPIKey(modelTestKey), "onechat-"+probeName, usagePath)

-       memory, err := openMemory(t.Context(), root, newTestClock().Now, newCompactor(client, probePrompt, defaultConfig().Compactor, promptTools()), discardLog())
+       memory, err := openMemory(t.Context(), root, newTestClock().Now, newCompactor(client, probePrompt, defaultConfig().Compactor, promptTools(), &ImageStore{dir: t.TempDir()}), discardLog())

        require.NoError(t, err)
 
        return &BuildRun{
        require.NoError(t, err)
 
        return &BuildRun{

diff --git a/chain.go b/chain.go
index c4a4302472d030ee944af5d7bc33f1e96ed7d9f1..23c7dec14a6053d8ec38d5201a06f6dc916b4532 100644 (file)
--- a/chain.go
+++ b/chain.go
@@ -32,6 +32,7 @@ type Chain struct {
        tools   *Toolbox
        request ModelRequest
        take    func() []Message
        tools   *Toolbox
        request ModelRequest
        take    func() []Message

+       images  *ImageStore

 }
 
 // Run runs the chain from input, the input of its first request, and returns
 }
 
 // Run runs the chain from input, the input of its first request, and returns


@@ -122,7 +123,12 @@ func (c *Chain) handle(ctx context.Context, input []any, response ModelResponse)
 
        input = append(input, outputs...)
        for _, message := range c.take() {
 
        input = append(input, outputs...)
        for _, message := range c.take() {

-               input = append(input, modelUserMessage(message.Text))
+               item, err := imageUserMessage(c.images, message.Text)
+               if err != nil {
+                       return nil, "", fmt.Errorf("read queued images: %w", err)
+               }
+
+               input = append(input, item)

        }
 
        return input, "", nil
        }
 
        return input, "", nil


@@ -141,7 +147,7 @@ type ChainFunctionCall struct {
 type ChainFunctionCallOutput struct {
        Type   string `json:"type"`
        CallID string `json:"call_id"`
 type ChainFunctionCallOutput struct {
        Type   string `json:"type"`
        CallID string `json:"call_id"`

-       Output string `json:"output"`
+       Output any    `json:"output"`

 }
 
 // chainInputItem returns an output item as an input item for the next request.
 }
 
 // chainInputItem returns an output item as an input item for the next request.

diff --git a/compactor.go b/compactor.go
index 8a16c6e8da18d3ff39ea07eed762e7bc57ee7f4d..2b239015ce763f56d1a23a4621647f056023b07b 100644 (file)
--- a/compactor.go
+++ b/compactor.go
@@ -4,7 +4,6 @@ import (
        "context"
        "errors"
        "fmt"
        "context"
        "errors"
        "fmt"

-       "slices"

        "strings"
        "unicode/utf8"
 )
        "strings"
        "unicode/utf8"
 )


@@ -20,12 +19,13 @@ var compactorRuler = strings.Repeat("-", nodeBytes)
 // client with model. instructions gives the system prompt of the agent at each
 // build. tools are the tools of that agent, so a build sends the same prompt
 // and tools as its turns.
 // client with model. instructions gives the system prompt of the agent at each
 // build. tools are the tools of that agent, so a build sends the same prompt
 // and tools as its turns.

-func newCompactor(client *ModelClient, instructions func() (string, error), model ModelConfig, tools []ModelTool) *Compactor {
+func newCompactor(client *ModelClient, instructions func() (string, error), model ModelConfig, tools []ModelTool, images *ImageStore) *Compactor {

        return &Compactor{
                client:       client,
                model:        model,
                instructions: instructions,
                tools:        tools,
        return &Compactor{
                client:       client,
                model:        model,
                instructions: instructions,
                tools:        tools,

+               images:       images,

        }
 }
 
        }
 }
 


@@ -37,17 +37,19 @@ func newCompactor(client *ModelClient, instructions func() (string, error), mode
 //
 // When the line is over nodeBytes, the next try sends the same message, plus
 // one more user message with a note with the size of the newest line and its
 //
 // When the line is over nodeBytes, the next try sends the same message, plus
 // one more user message with a note with the size of the newest line and its

-// cut at the limit. A try never sends the model's earlier reply or reasoning
-// back, so every try begins with the same view and task and has about the same
-// size. After at most compactorTries tries, the shortest line is the node. A
-// few bytes over the limit is fine, because the view measures the real sizes. A
-// call that fails before any line fails the build, and the core tries it again
-// at the next message.
+// cut at the limit. Only the first call of an image leaf sends its pixels. A
+// try never sends the model's earlier reply or reasoning back, so every try
+// begins with the same view and task and has about the same size. After at most
+// compactorTries tries, the shortest line is the node. A few bytes over the
+// limit is fine, because the view measures the real sizes. A call that fails
+// before any line fails the build, and the core tries it again at the next
+// message.

 type Compactor struct {
        client       *ModelClient
        model        ModelConfig
        instructions func() (string, error)
        tools        []ModelTool
 type Compactor struct {
        client       *ModelClient
        model        ModelConfig
        instructions func() (string, error)
        tools        []ModelTool

+       images       *ImageStore

 }
 
 // Summarize returns the line of the node of task.
 }
 
 // Summarize returns the line of the node of task.


@@ -62,7 +64,13 @@ func (c *Compactor) Summarize(ctx context.Context, task Task) (string, error) {
                return "", err
        }
 
                return "", err
        }
 

-       first := []any{modelUserMessage(task.Context + "\n" + text)}
+       source := task.Context + "\n" + text
+       parts, err := imageParts(c.images, source, task.Images)
+       if err != nil {
+               return "", fmt.Errorf("read compaction images: %w", err)
+       }
+
+       first := []any{modelUserParts(parts)}

 
        var lines []string
 
 
        var lines []string
 


@@ -70,7 +78,7 @@ func (c *Compactor) Summarize(ctx context.Context, task Task) (string, error) {
        for len(lines) < compactorTries {
                input := first
                if note != "" {
        for len(lines) < compactorTries {
                input := first
                if note != "" {

-                       input = append(slices.Clip(first), modelUserMessage(note))
+                       input = []any{modelUserMessage(source), modelUserMessage(note)}

                }
 
                var response ModelResponse
                }
 
                var response ModelResponse


@@ -111,12 +119,17 @@ func (c *Compactor) Summarize(ctx context.Context, task Task) (string, error) {
 // compactorTask returns the task of a build, with its input.
 func compactorTask(task Task) (string, error) {
        if task.Level == 0 && len(task.Input) == 1 {
 // compactorTask returns the task of a build, with its input.
 func compactorTask(task Task) (string, error) {
        if task.Level == 0 && len(task.Input) == 1 {

-               return fmt.Sprintf(`Compaction: compress message %d into one line of at most %d bytes
+               text := fmt.Sprintf(`Compaction: compress message %d into one line of at most %d bytes

 (about 70 words), the length of this ruler:
 %s
 <input>
 %s
 (about 70 words), the length of this ruler:
 %s
 <input>
 %s

-</input>`, task.Index, nodeBytes, compactorRuler, task.Input[0]), nil
+</input>`, task.Index, nodeBytes, compactorRuler, task.Input[0])
+               if len(task.Images) > 0 {
+                       text += "\nKeep each [image FILE] name and describe its kind, visible text, amounts, names, and context."
+               }
+
+               return text, nil

        }
 
        if task.Level > 0 && len(task.Input) == 2 {
        }
 
        if task.Level > 0 && len(task.Input) == 2 {

diff --git a/compactor_test.go b/compactor_test.go
index 854051574f6ed6218026a0e415fda5489883ba69..baa7f57ed7c6c88db4342403ea8f2093b793c5f5 100644 (file)
--- a/compactor_test.go
+++ b/compactor_test.go
@@ -208,7 +208,7 @@ func newTestCompactor(t *testing.T, server *ModelFake) *Compactor {
        return newCompactor(client, compactorAdaPrompt, ModelConfig{
                Model:  "gpt-test",
                Effort: "high",
        return newCompactor(client, compactorAdaPrompt, ModelConfig{
                Model:  "gpt-test",
                Effort: "high",

-       }, promptTools())
+       }, promptTools(), &ImageStore{dir: t.TempDir()})

 }
 
 // compactorInputTexts returns the text of each user message of call. Each
 }
 
 // compactorInputTexts returns the text of each user message of call. Each

diff --git a/daemon_agent.go b/daemon_agent.go
index 662291c637c14c208987e135316bf1f8e0141112..227d23b9c24865ca1f04e1b9e8f7e2e48871c786 100644 (file)
--- a/daemon_agent.go
+++ b/daemon_agent.go
@@ -55,7 +55,7 @@ func openDaemonAgents(ctx context.Context, dir, address string, auth ModelAuthor
 // one agent. All model calls use its cache key and usage file.
 func openDaemonAgent(ctx context.Context, agent *Agent, address string, auth ModelAuthorizer, compactor ModelConfig, clock func() time.Time, logger *slog.Logger) (*DaemonAgent, error) {
        client := newModelClient(address, auth, agent.CacheKey(), filepath.Join(agent.dir, "usage.jsonl"))
 // one agent. All model calls use its cache key and usage file.
 func openDaemonAgent(ctx context.Context, agent *Agent, address string, auth ModelAuthorizer, compactor ModelConfig, clock func() time.Time, logger *slog.Logger) (*DaemonAgent, error) {
        client := newModelClient(address, auth, agent.CacheKey(), filepath.Join(agent.dir, "usage.jsonl"))

-       memory, err := openMemory(ctx, agent.dir, clock, newCompactor(client, agent.Prompt, compactor, agent.Tools()), logger)
+       memory, err := openMemory(ctx, agent.dir, clock, newCompactor(client, agent.Prompt, compactor, agent.Tools(), agent.Images()), logger)

        if err != nil {
                return nil, fmt.Errorf("open memory: %w", err)
        }
        if err != nil {
                return nil, fmt.Errorf("open memory: %w", err)
        }

diff --git a/image_model.go b/image_model.go
new file mode 100644 (file)
index 0000000..6965142
--- /dev/null
+++ b/image_model.go
@@ -0,0 +1,90 @@
+package main
+
+import (
+       "encoding/base64"
+       "errors"
+       "fmt"
+       "strings"
+)
+
+// imageUserMessage attaches pixels only to messages selected as new by a turn.
+func imageUserMessage(store *ImageStore, text string) (any, error) {
+       names := imageFiles(text)
+       if len(names) == 0 {
+               return modelUserMessage(text), nil
+       }
+
+       parts, err := imageParts(store, text, names)
+       if err != nil {
+               return nil, fmt.Errorf("make image message: %w", err)
+       }
+
+       return modelUserParts(parts), nil
+}
+
+// imageParts is the shared wire format for image messages and zoom results.
+// Tool results use this list directly as function_call_output.output.
+func imageParts(store *ImageStore, text string, names []string) ([]ModelInputPart, error) {
+       parts := []ModelInputPart{{
+               Type: "input_text",
+               Text: text,
+       }}
+       for _, name := range names {
+               if store == nil {
+                       return nil, errors.New("image store is not available")
+               }
+
+               data, err := store.Read(name)
+               if err != nil {
+                       return nil, fmt.Errorf("read image part: %w", err)
+               }
+
+               mime := "image/" + strings.TrimPrefix(name[64:], ".")
+               if strings.HasSuffix(name, ".jpg") {
+                       mime = "image/jpeg"
+               }
+
+               parts = append(parts, ModelInputPart{
+                       Type:     "input_image",
+                       ImageURL: "data:" + mime + ";base64," + base64.StdEncoding.EncodeToString(data),
+               })
+       }
+
+       return parts, nil
+}
+
+// imageToolOutput leaves all results as text except zoom of one image message.
+// Only the text goes to the log, never the data URLs.
+func imageToolOutput(box *Toolbox, name, arguments, text string) (any, error) {
+       if name != "zoom" || strings.HasPrefix(text, "error:") {
+               return text, nil
+       }
+
+       var args ToolZoomArguments
+
+       err := toolDecodeArguments(arguments, &args)
+       if err != nil {
+               return nil, fmt.Errorf("decode image zoom: %w", err)
+       }
+
+       if args.Name != nil || args.N != 1 {
+               return text, nil
+       }
+
+       message, err := box.chat.Message(args.ID)
+       if err != nil {
+               return nil, fmt.Errorf("read image zoom: %w", err)
+       }
+
+       names := imageFiles(message.Text)
+       if len(names) == 0 {
+               return text, nil
+       }
+
+       parts, err := imageParts(box.images, text, names)
+       if err != nil {
+               return nil, fmt.Errorf("make image zoom: %w", err)
+       }
+
+       return parts, nil
+}

diff --git a/image_model_test.go b/image_model_test.go
new file mode 100644 (file)
index 0000000..53b0f31
--- /dev/null
+++ b/image_model_test.go
@@ -0,0 +1,199 @@
+package main
+
+import (
+       "encoding/base64"
+       "encoding/json"
+       "net/http"
+       "path/filepath"
+       "strings"
+       "testing"
+
+       "github.com/stretchr/testify/require"
+)
+
+func TestNewTurnKeepsImagesInEveryChainRequestButNotLaterTurns(t *testing.T) {
+       server := newModelFake(t)
+       server.reply = func(w http.ResponseWriter, call ModelFakeCall) {
+               if call.number == 0 {
+                       modelWriteEvents(w, modelCompletedEvent([]any{modelFunctionCallItem("fc_1", "call_1", "date", `{"id":0}`)}, 100, 0, 5))
+
+                       return
+               }
+
+               modelWriteEvents(w, modelCompletedEvent([]any{modelMessageItem("A red square.")}, 100, 0, 5))
+       }
+
+       test := newTestTurnLoop(t, server)
+       data := imageTestPNG(t, 3, 3)
+       name, err := test.agent.Images().Save(data)
+       require.NoError(t, err)
+
+       marker := imageText([]string{name}, "What is this?")
+       _, err = test.loop.Submit(terminalSource, marker)
+       require.NoError(t, err)
+       test.events.WaitFor(t, "idle", 1)
+       require.Equal(t, 2, server.Calls())
+
+       first := server.Call(0).body["input"].([]any)
+       second := server.Call(1).body["input"].([]any)
+       require.Equal(t, first, second[:len(first)])
+
+       expected := "data:image/png;base64," + base64.StdEncoding.EncodeToString(data)
+       require.Contains(t, string(server.Call(0).raw), expected)
+       require.Contains(t, string(server.Call(1).raw), expected)
+
+       _, err = test.loop.Submit(terminalSource, "Continue with text.")
+       require.NoError(t, err)
+       test.events.WaitFor(t, "idle", 2)
+       require.NotContains(t, string(server.Call(2).raw), "input_image")
+       require.NotContains(t, strings.Join(test.Log(t), "\n"), "base64")
+}
+
+func TestImageArrivingDuringAChainJoinsItsNextRequest(t *testing.T) {
+       server := newModelFake(t)
+       arrived := make(chan struct{})
+       release := make(chan struct{})
+       server.reply = func(w http.ResponseWriter, call ModelFakeCall) {
+               if call.number == 0 {
+                       close(arrived)
+                       <-release
+                       modelWriteEvents(w, modelCompletedEvent([]any{modelFunctionCallItem("fc_1", "call_1", "date", `{"id":0}`)}, 100, 0, 5))
+
+                       return
+               }
+
+               modelWriteEvents(w, modelCompletedEvent([]any{modelMessageItem("done")}, 100, 0, 5))
+       }
+
+       test := newTestTurnLoop(t, server)
+       _, err := test.loop.Submit(terminalSource, "Start.")
+       require.NoError(t, err)
+       <-arrived
+
+       name, err := test.agent.Images().Save(imageTestPNG(t, 2, 2))
+       require.NoError(t, err)
+
+       _, err = test.loop.Submit(terminalSource, imageText([]string{name}, "Read this."))
+       require.NoError(t, err)
+       close(release)
+       test.events.WaitFor(t, "idle", 1)
+       require.NotContains(t, string(server.Call(0).raw), "input_image")
+       require.Contains(t, string(server.Call(1).raw), "input_image")
+}
+
+func TestLeafCompactionDescribesImagesOnceAndMergesOnlyText(t *testing.T) {
+       server := newModelFake(t)
+       store := &ImageStore{dir: filepath.Join(t.TempDir(), imageDirName)}
+       name, err := store.Save(imageTestPNG(t, 2, 2))
+       require.NoError(t, err)
+
+       server.reply = func(w http.ResponseWriter, call ModelFakeCall) {
+               text := "receipt, Ada, $12, red square"
+               if call.number == 0 {
+                       text = strings.Repeat("detail ", 100)
+               }
+
+               modelWriteEvents(w, modelCompletedEvent([]any{modelMessageItem(text)}, 100, 0, 5))
+       }
+
+       compactor := newTestCompactor(t, server)
+       compactor.images = store
+       task := Task{
+               Level:   0,
+               Index:   0,
+               Context: "<chat>\n</chat>",
+               Input:   []string{"user: " + imageText([]string{name}, "Read the receipt.")},
+               Images:  []string{name},
+       }
+       text, err := compactor.Summarize(t.Context(), task)
+       require.NoError(t, err)
+       require.Equal(t, "receipt, Ada, $12, red square", text)
+       require.Contains(t, string(server.Call(0).raw), "input_image")
+       require.Contains(t, string(server.Call(0).raw), "kind, visible text, amounts, names, and context")
+       require.NotContains(t, string(server.Call(1).raw), "input_image")
+
+       _, err = compactor.Summarize(t.Context(), Task{
+               Level:   1,
+               Index:   0,
+               Context: "<chat>\n</chat>",
+               Input:   []string{text, "user: next"},
+       })
+       require.NoError(t, err)
+       require.NotContains(t, string(server.Call(2).raw), "input_image")
+
+       memory, err := openMemory(t.Context(), t.TempDir(), newTestClock().Now, compactor, discardLog())
+       require.NoError(t, err)
+       t.Cleanup(func() {
+               require.NoError(t, memory.Close())
+       })
+
+       _, err = memory.Append("user", imageText([]string{name}, "A short caption."))
+       require.NoError(t, err)
+       require.NoError(t, memory.Settle(t.Context()))
+       require.Equal(t, 4, server.Calls())
+       require.Contains(t, string(server.Call(3).raw), "input_image")
+       require.Contains(t, memory.View(), "receipt, Ada, $12")
+}
+
+func TestZoomReturnsImagePartsButLogsOnlyText(t *testing.T) {
+       server := newModelFake(t)
+       server.reply = func(w http.ResponseWriter, call ModelFakeCall) {
+               if call.number == 0 {
+                       modelWriteEvents(w, modelCompletedEvent([]any{modelFunctionCallItem("fc_1", "call_1", "zoom", `{"id":0,"n":1,"name":null}`)}, 100, 0, 5))
+
+                       return
+               }
+
+               modelWriteEvents(w, modelCompletedEvent([]any{modelMessageItem("A red square.")}, 100, 0, 5))
+       }
+
+       test := newTestTurnLoop(t, server)
+       name, err := test.agent.Images().Save(imageTestPNG(t, 2, 2))
+       require.NoError(t, err)
+
+       _, err = test.memory.Append("user", imageText([]string{name}, "Old image."))
+       require.NoError(t, err)
+
+       _, err = test.loop.Submit(terminalSource, "Zoom the old image.")
+       require.NoError(t, err)
+       test.events.WaitFor(t, "idle", 1)
+       require.NotContains(t, string(server.Call(0).raw), "input_image")
+
+       input := server.Call(1).body["input"].([]any)
+       result := input[len(input)-1].(map[string]any)
+       require.Equal(t, "function_call_output", result["type"])
+
+       parts := result["output"].([]any)
+       require.Len(t, parts, 2)
+       require.Equal(t, "input_text", parts[0].(map[string]any)["type"])
+       require.Equal(t, "input_image", parts[1].(map[string]any)["type"])
+       require.Contains(t, parts[0].(map[string]any)["text"], "Old image.")
+       require.NotContains(t, strings.Join(test.Log(t), "\n"), "base64")
+}
+
+func TestImageUserMessageKeepsTextPartsAndMarkerOrder(t *testing.T) {
+       store := &ImageStore{dir: t.TempDir()}
+       first, err := store.Save(imageTestPNG(t, 1, 2))
+       require.NoError(t, err)
+
+       second, err := store.Save(imageTestPNG(t, 2, 1))
+       require.NoError(t, err)
+
+       text := imageText([]string{first, second}, "Compare.")
+       item, err := imageUserMessage(store, text)
+       require.NoError(t, err)
+
+       data, err := json.Marshal(item)
+       require.NoError(t, err)
+
+       var message struct {
+               Role    string           `json:"role"`
+               Content []ModelInputPart `json:"content"`
+       }
+
+       require.NoError(t, json.Unmarshal(data, &message))
+       require.Equal(t, "user", message.Role)
+       require.Len(t, message.Content, 3)
+       require.Equal(t, text, message.Content[0].Text)
+       require.NotEqual(t, message.Content[1].ImageURL, message.Content[2].ImageURL)
+}

diff --git a/memory.go b/memory.go
index a240b6c16d1d898e3051f3e0a0c7a19434061b3e..602b86cddf5b813027c554a23f65621298f46fc3 100644 (file)
--- a/memory.go
+++ b/memory.go
@@ -607,6 +607,7 @@ func (m *Memory) task(key NodeKey) (Task, error) {
                }
 
                task.Input = []string{leafSource(message)}
                }
 
                task.Input = []string{leafSource(message)}

+               task.Images = imageFiles(message.Text)

 
                return task, nil
        }
 
                return task, nil
        }


@@ -781,12 +782,14 @@ func (m *Memory) buildBuiltByModel(key NodeKey) (bool, error) {
 
 // Task is the build of the node (Level, Index). Context is the compaction view
 // for the build. Input holds the source "kind: text" of a leaf, or the two
 
 // Task is the build of the node (Level, Index). Context is the compaction view
 // for the build. Input holds the source "kind: text" of a leaf, or the two

-// child lines of a merge with their newlines as spaces.
+// child lines of a merge with their newlines as spaces. Images holds the leaf
+// image filenames and is empty for merges.

 type Task struct {
        Level   int
        Index   int
        Context string
        Input   []string
 type Task struct {
        Level   int
        Index   int
        Context string
        Input   []string

+       Images  []string

 }
 
 // MemoryViewFile is the content of view.json.
 }
 
 // MemoryViewFile is the content of view.json.

diff --git a/model.go b/model.go
index 507fbb1a260265dbdd0f61c817ba33aef07d9cfa..6ddd3008cca12c96577ccc5a9a8b6ca913ebfe7e 100644 (file)
--- a/model.go
+++ b/model.go
@@ -73,22 +73,22 @@ func newModelClient(url string, auth ModelAuthorizer, cacheKey, usagePath string
 
 // modelUserMessage returns a user input item with one text part for each text.
 func modelUserMessage(texts ...string) any {
 
 // modelUserMessage returns a user input item with one text part for each text.
 func modelUserMessage(texts ...string) any {

-       type part struct {
-               Type string `json:"type"`
-               Text string `json:"text"`
-       }
-
-       content := make([]part, len(texts))
+       content := make([]ModelInputPart, len(texts))

        for n, text := range texts {
        for n, text := range texts {

-               content[n] = part{
+               content[n] = ModelInputPart{

                        Type: "input_text",
                        Text: text,
                }
        }
 
                        Type: "input_text",
                        Text: text,
                }
        }
 

+       return modelUserParts(content)
+}
+
+// modelUserParts keeps text and image parts in one user message.
+func modelUserParts(content []ModelInputPart) any {

        return struct {
        return struct {

-               Role    string `json:"role"`
-               Content []part `json:"content"`
+               Role    string           `json:"role"`
+               Content []ModelInputPart `json:"content"`

        }{
                Role:    "user",
                Content: content,
        }{
                Role:    "user",
                Content: content,


@@ -321,6 +321,13 @@ type ModelOutputItem struct {
        } `json:"content"`
 }
 
        } `json:"content"`
 }
 

+// ModelInputPart is a text or image part in a user message or tool result.
+type ModelInputPart struct {
+       Type     string `json:"type"`
+       Text     string `json:"text,omitempty"`
+       ImageURL string `json:"image_url,omitempty"`
+}
+

 // ModelRequest is one model call. Input holds the input items in order, such as
 // modelUserMessage values or output items of an earlier response. Node names
 // the node that a compaction builds, or is nil. Turn names the request of a
 // ModelRequest is one model call. Input holds the input items in order, such as
 // modelUserMessage values or output items of an earlier response. Node names
 // the node that a compaction builds, or is nil. Turn names the request of a

diff --git a/probe.go b/probe.go
index b00482952bdb10b18d47dcaff524805ed85c089e..384d0f8357787e15fe5c5ccf07b9aa51bb0c20c5 100644 (file)
--- a/probe.go
+++ b/probe.go
@@ -103,7 +103,7 @@ func probe(ctx context.Context, args []string, out io.Writer) (err error) {
        client.LimitCalls(probeCallLimit)
 
        summarizer := ProbeSlotSummarizer{
        client.LimitCalls(probeCallLimit)
 
        summarizer := ProbeSlotSummarizer{

-               inner: newCompactor(client, probePrompt, config.Compactor, promptTools()),
+               inner: newCompactor(client, probePrompt, config.Compactor, promptTools(), &ImageStore{dir: filepath.Join(dir, imageDirName)}),

                slots: make(chan struct{}, builds),
        }
 
                slots: make(chan struct{}, builds),
        }
 

diff --git a/prompt.go b/prompt.go
index 36fda3288de8778c156fc51f35e79122b1015725..df6f17d4a0c898d6ee2a0a448a916d85c0caf3c6 100644 (file)
--- a/prompt.go
+++ b/prompt.go
@@ -41,10 +41,10 @@ var promptEfforts = []string{"low", "medium", "high", "xhigh"}
 // one added paragraph on the user's instructions file. The reference names its
 // own agent, so promptText has the name of the agent and the kind reply in its
 // place. It describes only what OneChat can do, so it leaves out the spec text
 // one added paragraph on the user's instructions file. The reference names its
 // own agent, so promptText has the name of the agent and the kind reply in its
 // place. It describes only what OneChat can do, so it leaves out the spec text

-// on images, computers, and computer tasks. Its paragraph on persistent mode
-// tells the model that a final answer ends the turn. prompt adds the clause on
-// the instructions of the user only when there are instructions. It holds no
-// date and no state, so turns and compactions share its prompt cache.
+// on computers and computer tasks. Its paragraph on persistent mode tells the
+// model that a final answer ends the turn. prompt adds the clause on the
+// instructions of the user only when there are instructions. It holds no date
+// and no state, so turns and compactions share its prompt cache.

 const promptText = `You are {name}, an AI agent that works for one user in a single chat that never
 ends. Each call to you is a turn or a compaction: the view below is followed by
 the user's new message, or by a task starting "Compaction:".
 const promptText = `You are {name}, an AI agent that works for one user in a single chat that never
 ends. Each call to you is a turn or a compaction: the view below is followed by
 the user's new message, or by a task starting "Compaction:".


@@ -76,6 +76,9 @@ Tools:
 - zoom("Name") gives an agent's whole chat
 - date(id) gives the date and time of message id
 
 - zoom("Name") gives an agent's whole chat
 - date(id) gives the date and time of message id
 

+An [image FILE] line names a stored image. New images include pixels for this
+turn only. Use zoom(id, 1) to see an older image again.
+

 # Turns
 
 Do the user's tasks yourself, with your tools.
 # Turns
 
 Do the user's tasks yourself, with your tools.


@@ -155,10 +158,10 @@ Set or remove a route only after the user agrees.`
 
 // promptSubagentText is the system prompt of a subagent, from the subagent
 // prompt of the reference design: its head, then the doc of the view, without
 
 // promptSubagentText is the system prompt of a subagent, from the subagent
 // prompt of the reference design: its head, then the doc of the view, without

-// the parts on devices and images. A subagent gets the view of the agent, then
-// its whole log, so it goes on from its log after a tell or a restart.
-// promptSubagent adds the clause on the instructions of the user only when
-// there are instructions.
+// the parts on devices. A subagent gets the view of the agent, then its whole
+// log, so it goes on from its log after a tell or a restart. promptSubagent
+// adds the clause on the instructions of the user only when there are
+// instructions.

 const promptSubagentText = `You are a subagent of {name}, an AI agent that works for one user in a single
 chat that never ends. {name} gave you a task. Do it yourself, with your tools.
 
 const promptSubagentText = `You are a subagent of {name}, an AI agent that works for one user in a single
 chat that never ends. {name} gave you a task. Do it yourself, with your tools.
 


@@ -189,7 +192,8 @@ was made from; zoom(id, 1) gives message id in full. Zoom whenever a summary
 only mentions something you need, such as what your last reply said, a
 decision, a past attempt or where a file is, before you act, guess or ask.
 date(id) gives the date and time of message id. zoom("Name") gives an agent's
 only mentions something you need, such as what your last reply said, a
 decision, a past attempt or where a file is, before you act, guess or ask.
 date(id) gives the date and time of message id. zoom("Name") gives an agent's

-whole chat.`
+whole chat. An [image FILE] line names a stored image. Use zoom(id, 1)
+to see its pixels.`

 
 // prompt returns the system prompt of the reference with one added paragraph on
 // the user's instructions file at instructionsPath. It uses the agent name and
 
 // prompt returns the system prompt of the reference with one added paragraph on
 // the user's instructions file at instructionsPath. It uses the agent name and

diff --git a/prompt_test.go b/prompt_test.go
index c9cadc2167022026836b29e6259992114c22afab..e9e87f347b9f2676f5ba5cc8ba46f7c0dd83352f 100644 (file)
--- a/prompt_test.go
+++ b/prompt_test.go
@@ -18,6 +18,7 @@ func TestPromptNamesTheAgent(t *testing.T) {
        require.Contains(t, text, `- work: an agent's report, starting "[Name]"`)
        require.Contains(t, text, "- zoom(id, 1) gives message id whole\n- zoom(\"Name\") gives an agent's whole chat\n")
        require.NotContains(t, text, promptName)
        require.Contains(t, text, `- work: an agent's report, starting "[Name]"`)
        require.Contains(t, text, "- zoom(id, 1) gives message id whole\n- zoom(\"Name\") gives an agent's whole chat\n")
        require.NotContains(t, text, promptName)

+       require.Contains(t, text, "[image FILE]")

 
        // The old agent name is built from parts, so the source does not spell
        // it.
 
        // The old agent name is built from parts, so the source does not spell
        // it.


@@ -26,7 +27,7 @@ func TestPromptNamesTheAgent(t *testing.T) {
 
 func TestPromptLeavesOutAbsentFeatures(t *testing.T) {
        text := prompt("Ada", "/AGENT/instructions.md", "")
 
 func TestPromptLeavesOutAbsentFeatures(t *testing.T) {
        text := prompt("Ada", "/AGENT/instructions.md", "")

-       for _, absent := range []string{"image", "computer", "device", "instructions at the end"} {
+       for _, absent := range []string{"computer", "device", "instructions at the end"} {

                require.NotContains(t, text, absent)
        }
 }
                require.NotContains(t, text, absent)
        }
 }


@@ -50,7 +51,7 @@ func TestSubagentPromptNamesTheAgentAndEndsWithTheInstructions(t *testing.T) {
        require.True(t, strings.HasSuffix(text, "\n\n# The user's instructions\n\nI am Lena."))
        require.NotContains(t, text, promptName)
 
        require.True(t, strings.HasSuffix(text, "\n\n# The user's instructions\n\nI am Lena."))
        require.NotContains(t, text, promptName)
 

-       for _, absent := range []string{"image", "computer", "device", "Compaction"} {
+       for _, absent := range []string{"computer", "device", "Compaction"} {

                require.NotContains(t, text, absent)
        }
 
                require.NotContains(t, text, absent)
        }
 

diff --git a/rebuild_test.go b/rebuild_test.go
index fc4ff7e8b09dc3cfd23d46c221cc7e36fbce8fcd..104ec88072bc310ef1dd76fd68fbccccdf9cb392 100644 (file)
--- a/rebuild_test.go
+++ b/rebuild_test.go
@@ -63,7 +63,7 @@ func TestRebuildMergesDatesAndKeepsAgentFiles(t *testing.T) {
                },
        }
 
                },
        }
 

-       backup, err := rebuild(t.Context(), agent, added, client, newCompactor(client, agent.Prompt, defaultConfig().Compactor, agent.Tools()), &report)
+       backup, err := rebuild(t.Context(), agent, added, client, newCompactor(client, agent.Prompt, defaultConfig().Compactor, agent.Tools(), agent.Images()), &report)

        require.NoError(t, err)
        require.Equal(t, before, rebuildTestSnapshot(t, backup))
 
        require.NoError(t, err)
        require.Equal(t, before, rebuildTestSnapshot(t, backup))
 


@@ -143,7 +143,7 @@ func TestRebuildFailureAndCancellationKeepOriginal(t *testing.T) {
                                Kind: "work",
                                Text: strings.Repeat("long ", 200),
                                Date: "2025-01-01T00:00:00Z",
                                Kind: "work",
                                Text: strings.Repeat("long ", 200),
                                Date: "2025-01-01T00:00:00Z",

-                       }}, client, newCompactor(client, agent.Prompt, defaultConfig().Compactor, agent.Tools()), &strings.Builder{})
+                       }}, client, newCompactor(client, agent.Prompt, defaultConfig().Compactor, agent.Tools(), agent.Images()), &strings.Builder{})

                        require.Error(t, err)
                        require.Empty(t, backup)
                        require.Equal(t, before, rebuildTestSnapshot(t, agent.dir))
                        require.Error(t, err)
                        require.Empty(t, backup)
                        require.Equal(t, before, rebuildTestSnapshot(t, agent.dir))


@@ -179,7 +179,7 @@ func TestRebuildKeepsOriginalWhenAnInvisibleMergeFails(t *testing.T) {
        }
 
        client := newModelClient(server.URL, ModelAPIKey(modelTestKey), agent.CacheKey(), filepath.Join(t.TempDir(), "usage.jsonl"))
        }
 
        client := newModelClient(server.URL, ModelAPIKey(modelTestKey), agent.CacheKey(), filepath.Join(t.TempDir(), "usage.jsonl"))

-       _, err := rebuild(t.Context(), agent, nil, client, newCompactor(client, agent.Prompt, defaultConfig().Compactor, agent.Tools()), &strings.Builder{})
+       _, err := rebuild(t.Context(), agent, nil, client, newCompactor(client, agent.Prompt, defaultConfig().Compactor, agent.Tools(), agent.Images()), &strings.Builder{})

        require.ErrorContains(t, err, "summary tree has unfinished nodes")
        require.Equal(t, before, rebuildTestSnapshot(t, agent.dir))
 }
        require.ErrorContains(t, err, "summary tree has unfinished nodes")
        require.Equal(t, before, rebuildTestSnapshot(t, agent.dir))
 }

diff --git a/subagent_run.go b/subagent_run.go
index 8084c390a7012d5213019f1f784ec3214ce08bbe..891766c4e0baf5895b81617a0ede42eea19e13d6 100644 (file)
--- a/subagent_run.go
+++ b/subagent_run.go
@@ -515,11 +515,13 @@ func (r *SubagentRuns) chain(ctx context.Context, run *SubagentRun) (string, err
        input = append(input, modelUserMessage(r.state()), modelUserMessage(log))
        chain := &Chain{
                client: r.client,
        input = append(input, modelUserMessage(r.state()), modelUserMessage(log))
        chain := &Chain{
                client: r.client,

+               images: r.agent.Images(),

                tools: &Toolbox{
                        log:       run.subagent,
                        chat:      r.memory,
                        subagents: r.subagents,
                        workdir:   run.subagent.WorkDir(),
                tools: &Toolbox{
                        log:       run.subagent,
                        chat:      r.memory,
                        subagents: r.subagents,
                        workdir:   run.subagent.WorkDir(),

+                       images:    r.agent.Images(),

                },
                request: ModelRequest{
                        Model:        r.agent.config.Model,
                },
                request: ModelRequest{
                        Model:        r.agent.config.Model,

diff --git a/tools.go b/tools.go
index a75514f984964224f0e1075d06281cda3be79cb0..96cbb97d38010e9d6e53871b7bef0c471dba5dcf 100644 (file)
--- a/tools.go
+++ b/tools.go
@@ -68,13 +68,15 @@ type Toolbox struct {
        messages  *DaemonMessages
        children  *DaemonChildren
        routes    *Routes
        messages  *DaemonMessages
        children  *DaemonChildren
        routes    *Routes

+       images    *ImageStore

 }
 
 // Call logs the call of the tool name with its JSON arguments, makes the log
 }
 
 // Call logs the call of the tool name with its JSON arguments, makes the log

-// durable, runs the tool, and logs its clipped output, which it returns. A
-// failed tool gives its error as the output. An error of Call itself is an
-// error of the log. When ctx ends before the flush, the tool does not run.
-func (b *Toolbox) Call(ctx context.Context, name, arguments string) (string, error) {
+// durable, runs the tool, and logs its clipped text. It returns that text or
+// the text and image parts of a single-message zoom. Pixels never enter the
+// log. A failed tool gives its error as the output. An error of Call itself is
+// an error of the log. When ctx ends before the flush, the tool does not run.
+func (b *Toolbox) Call(ctx context.Context, name, arguments string) (any, error) {

        _, err := b.log.Append("tool", toolCallText(name, arguments))
        if err != nil {
                return "", fmt.Errorf("append tool call: %w", err)
        _, err := b.log.Append("tool", toolCallText(name, arguments))
        if err != nil {
                return "", fmt.Errorf("append tool call: %w", err)


@@ -90,12 +92,26 @@ func (b *Toolbox) Call(ctx context.Context, name, arguments string) (string, err
                output = toolClip(b.run(ctx, name, arguments))
        }
 
                output = toolClip(b.run(ctx, name, arguments))
        }
 

+       var (
+               result    any = output
+               resultErr error
+       )
+
+       if err == nil {
+               result, resultErr = imageToolOutput(b, name, arguments, output)
+       }
+
+       if resultErr != nil {
+               output = "error: " + resultErr.Error()
+               result = output
+       }
+

        _, err = b.log.Append("echo", output)
        if err != nil {
                return "", fmt.Errorf("append tool result: %w", err)
        }
 
        _, err = b.log.Append("echo", output)
        if err != nil {
                return "", fmt.Errorf("append tool result: %w", err)
        }
 

-       return output, nil
+       return result, nil

 }
 
 // run runs one tool and returns its output, or its error as text.
 }
 
 // run runs one tool and returns its output, or its error as text.

diff --git a/tree.go b/tree.go
index 69558c54ca07197e5e09147092caa7a47127efdf..079969c26b6669329f28c4086b07695b1463d341 100644 (file)
--- a/tree.go
+++ b/tree.go
@@ -55,11 +55,11 @@ func leafSource(message Message) string {
        return message.Kind + ": " + message.Text
 }
 
        return message.Kind + ": " + message.Text
 }
 

-// freeLeaf returns the leaf of message when its source fits in a node. Such a
-// leaf holds the source word for word and needs no model call.
+// freeLeaf returns a text-only leaf when its source fits in a node. Images need
+// a model description even when their text fits.

 func freeLeaf(message Message) (Node, bool) {
        text := leafSource(message)
 func freeLeaf(message Message) (Node, bool) {
        text := leafSource(message)

-       if len(text) > nodeBytes {
+       if len(text) > nodeBytes || len(imageFiles(message.Text)) > 0 {

                return Node{}, false
        }
 
                return Node{}, false
        }
 

diff --git a/turn.go b/turn.go
index e0bd489f323549a19f19706d4e68fd04c656c762..2c1d9c22d11615aebaa14baa6f424812cbf3609c 100644 (file)
--- a/turn.go
+++ b/turn.go
@@ -60,6 +60,7 @@ func newTurnLoop(agent *Agent, memory *Memory, subagents *Subagents, whatsapp *W
                schedules: schedules,
                workdir:   agent.config.Workdir,
                agentName: agent.config.Name,
                schedules: schedules,
                workdir:   agent.config.Workdir,
                agentName: agent.config.Name,

+               images:    agent.Images(),

        }
 
        return l, nil
        }
 
        return l, nil


@@ -423,10 +424,19 @@ func (l *TurnLoop) chain(ctx context.Context) (string, error) {
                        Instructions: instructions,
                        Tools:        l.agent.Tools(),
                },
                        Instructions: instructions,
                        Tools:        l.agent.Tools(),
                },

-               take: l.take,
+               take:   l.take,
+               images: l.agent.Images(),

        }
 
        }
 

-       reply, err := chain.Run(ctx, turnInput(view, l.state(), messages))
+       input := turnInput(view, l.state(), messages)
+       for n, message := range messages {
+               input[len(input)-len(messages)+n], err = imageUserMessage(l.agent.Images(), message.Text)
+               if err != nil {
+                       return "", fmt.Errorf("read turn images: %w", err)
+               }
+       }
+
+       reply, err := chain.Run(ctx, input)

        if err != nil {
                return "", fmt.Errorf("run turn chain: %w", err)
        }
        if err != nil {
                return "", fmt.Errorf("run turn chain: %w", err)
        }

diff --git a/whatsapp_import.go b/whatsapp_import.go
index df871c19670029756c94f5e4d26e72e4ace9549a..1092b4924c0d7640450e433a75ecf8ceefd5ddf8 100644 (file)
--- a/whatsapp_import.go
+++ b/whatsapp_import.go
@@ -143,7 +143,7 @@ func whatsappImport(ctx context.Context, args []string, out io.Writer) (err erro
        }
 
        client := newModelClient(address, auth, agent.CacheKey(), usagePath)
        }
 
        client := newModelClient(address, auth, agent.CacheKey(), usagePath)

-       backup, err := rebuild(ctx, agent, plan.added, client, newCompactor(client, agent.Prompt, config.Compactor, agent.Tools()), out)
+       backup, err := rebuild(ctx, agent, plan.added, client, newCompactor(client, agent.Prompt, config.Compactor, agent.Tools(), agent.Images()), out)

        if err != nil {
                return fmt.Errorf("rebuild imported memory: %w", err)
        }
        if err != nil {
                return fmt.Errorf("rebuild imported memory: %w", err)
        }

diff --git a/whatsapp_import_test.go b/whatsapp_import_test.go
index 1f9c9afc23de996095d84a93856be679b9ac17f9..4bd0a1f3354db5b41b2bab288f9af598cd4343bd 100644 (file)
--- a/whatsapp_import_test.go
+++ b/whatsapp_import_test.go
@@ -563,7 +563,7 @@ func TestWhatsAppImportChildRebuildUsesOwnPromptAndCache(t *testing.T) {
        }
 
        client := newModelClient(server.URL, ModelAPIKey(modelTestKey), child.CacheKey(), filepath.Join(t.TempDir(), "usage.jsonl"))
        }
 
        client := newModelClient(server.URL, ModelAPIKey(modelTestKey), child.CacheKey(), filepath.Join(t.TempDir(), "usage.jsonl"))

-       backup, err := rebuild(t.Context(), child, plan.added, client, newCompactor(client, child.Prompt, defaultConfig().Compactor, child.Tools()), &strings.Builder{})
+       backup, err := rebuild(t.Context(), child, plan.added, client, newCompactor(client, child.Prompt, defaultConfig().Compactor, child.Tools(), child.Images()), &strings.Builder{})

        require.NoError(t, err)
        require.Equal(t, before, rebuildTestSnapshot(t, backup))
        require.Equal(t, root.dir, filepath.Dir(backup))
        require.NoError(t, err)
        require.Equal(t, before, rebuildTestSnapshot(t, backup))
        require.Equal(t, root.dir, filepath.Dir(backup))


@@ -613,7 +613,7 @@ func TestWhatsAppImportChildFailureKeepsConfiguredChildren(t *testing.T) {
                Kind: "work",
                Text: strings.Repeat("The plan is ready. ", 80),
                Date: "2025-01-01T08:00:00Z",
                Kind: "work",
                Text: strings.Repeat("The plan is ready. ", 80),
                Date: "2025-01-01T08:00:00Z",

-       }}, client, newCompactor(client, child.Prompt, defaultConfig().Compactor, child.Tools()), &strings.Builder{})
+       }}, client, newCompactor(client, child.Prompt, defaultConfig().Compactor, child.Tools(), child.Images()), &strings.Builder{})

        require.ErrorContains(t, err, "rebuild")
        require.Equal(t, before, rebuildTestSnapshot(t, root.dir))
 
        require.ErrorContains(t, err, "rebuild")
        require.Equal(t, before, rebuildTestSnapshot(t, root.dir))