Show stored images to the user

The root can show a stored image as a Telegram photo and return its name
and path to terminal clients. A refused photo goes as a document, while
an uncertain send does not cause a second send. Children and subagents
have no image output tool.

diff --git a/agent.go b/agent.go
index 97d668585dd42d36e7816c3d2870e0d950b613d0..b86880e46ab465ae820303544479e4574412797e 100644 (file)
--- a/agent.go
+++ b/agent.go
@@ -117,12 +117,12 @@ func (a *Agent) Prompt() (string, error) {
 }
 
 // Tools returns the tools shared by the turns and compactions of this agent. A
 }
 
 // Tools returns the tools shared by the turns and compactions of this agent. A

-// child has no WhatsApp, child creation, or route tools.
+// child has no image output, WhatsApp, child creation, or route tools.

 func (a *Agent) Tools() []ModelTool {
        tools := promptTools()
        if a.config.Parent != "" {
                tools = slices.DeleteFunc(tools, func(tool ModelTool) bool {
 func (a *Agent) Tools() []ModelTool {
        tools := promptTools()
        if a.config.Parent != "" {
                tools = slices.DeleteFunc(tools, func(tool ModelTool) bool {

-                       return strings.HasPrefix(tool.Name, "whatsapp_") || tool.Name == "agent_create" || tool.Name == "route" || tool.Name == "unroute" || tool.Name == "routes"
+                       return strings.HasPrefix(tool.Name, "whatsapp_") || tool.Name == "image_show" || tool.Name == "agent_create" || tool.Name == "route" || tool.Name == "unroute" || tool.Name == "routes"

                })
        }
 
                })
        }
 

diff --git a/daemon.go b/daemon.go
index 137cc6f42a8e833ec5da6a6481847356b20ca565..5c94a4f25daff64fbc3d2339e0451bb46d43edf8 100644 (file)
--- a/daemon.go
+++ b/daemon.go
@@ -353,6 +353,10 @@ func daemonServe(ctx context.Context, listener net.Listener, bot *TelegramBot, w
                return errors.Join(err, listener.Close())
        }
 
                return errors.Join(err, listener.Close())
        }
 

+       if bot != nil {
+               l.tools.imageShow.bot = bot
+       }
+

        loops := []*TurnLoop{l}
        for _, child := range children {
                loop, err := child.Loop(nil, func(Event) {
        loops := []*TurnLoop{l}
        for _, child := range children {
                loop, err := child.Loop(nil, func(Event) {

diff --git a/daemon_children_test.go b/daemon_children_test.go
index 16f326fa6ac87168087819272544e2284b483ae5..b9e212f8646646473ee08512d4c54593c7e89632 100644 (file)
--- a/daemon_children_test.go
+++ b/daemon_children_test.go
@@ -335,6 +335,8 @@ func TestChildToolsExcludeCreationAndWhatsApp(t *testing.T) {
 
        names := promptToolNames(child.agent.Tools())
        require.NotContains(t, names, "agent_create")
 
        names := promptToolNames(child.agent.Tools())
        require.NotContains(t, names, "agent_create")

+       require.NotContains(t, names, "image_show")
+       require.Equal(t, `error: unknown tool "image_show"`, child.tools.run(t.Context(), "image_show", `{"image":"unknown","caption":""}`))

        require.NotContains(t, names, "whatsapp_send")
        require.NotContains(t, names, "whatsapp_find")
        require.Contains(t, names, "spawn")
        require.NotContains(t, names, "whatsapp_send")
        require.NotContains(t, names, "whatsapp_find")
        require.Contains(t, names, "spawn")

diff --git a/image_show.go b/image_show.go
new file mode 100644 (file)
index 0000000..80204c1
--- /dev/null
+++ b/image_show.go
@@ -0,0 +1,37 @@
+package main
+
+import (
+       "context"
+       "fmt"
+       "path/filepath"
+)
+
+// ImageShow shows a stored image to the root's user. A configured bot sends it
+// to its owner, and the tool result gives terminal clients its name and path.
+type ImageShow struct {
+       store *ImageStore
+       bot   *TelegramBot
+}
+
+// Show checks the stored name and bytes before any output. A refused Telegram
+// photo goes as a document. An uncertain send is not repeated.
+func (s *ImageShow) Show(ctx context.Context, name, caption string) (string, error) {
+       data, err := s.store.Read(name)
+       if err != nil {
+               return "", fmt.Errorf("read image to show: %w", err)
+       }
+
+       path, err := filepath.Abs(filepath.Join(s.store.dir, name))
+       if err != nil {
+               return "", fmt.Errorf("find image path: %w", err)
+       }
+
+       if s.bot != nil {
+               err = s.bot.api.SendImage(ctx, s.bot.owner, name, data, caption)
+               if err != nil {
+                       return "", fmt.Errorf("show Telegram image: %w", err)
+               }
+       }
+
+       return imageText([]string{name}, caption) + "\n" + path, nil
+}

diff --git a/image_show_test.go b/image_show_test.go
new file mode 100644 (file)
index 0000000..ed475fe
--- /dev/null
+++ b/image_show_test.go
@@ -0,0 +1,136 @@
+package main
+
+import (
+       "encoding/json"
+       "net/http"
+       "path/filepath"
+       "testing"
+
+       "github.com/stretchr/testify/require"
+)
+
+func TestImageShowUsesStoredImageForTelegramAndTerminal(t *testing.T) {
+       picture := imageTestPNG(t, 5, 4)
+       store := &ImageStore{dir: filepath.Join(t.TempDir(), imageDirName)}
+       name, err := store.Save(picture)
+       require.NoError(t, err)
+
+       telegram := newTelegramFake(t)
+       bot, logs := telegramStartTestBot(t, telegram)
+       box := &Toolbox{imageShow: &ImageShow{
+               store: store,
+               bot:   bot,
+       }}
+       args, err := json.Marshal(ToolImageShowArguments{
+               Image:   name,
+               Caption: "A red point.",
+       })
+       require.NoError(t, err)
+
+       output := box.run(t.Context(), "image_show", string(args))
+       require.Equal(t, imageText([]string{name}, "A red point.")+"\n"+filepath.Join(store.dir, name), output)
+
+       calls := telegram.Calls()
+       require.Len(t, calls, 1)
+       require.Equal(t, "sendPhoto", calls[0].method)
+       require.Equal(t, "4242", calls[0].params["chat_id"])
+       require.Equal(t, "A red point.", calls[0].params["caption"])
+       require.Equal(t, name, calls[0].params["photo_name"])
+       require.Equal(t, picture, calls[0].params["photo"])
+       require.Empty(t, logs.String())
+}
+
+func TestImageShowRefusesUnknownNamesBeforeSending(t *testing.T) {
+       telegram := newTelegramFake(t)
+       bot, _ := telegramStartTestBot(t, telegram)
+       show := &ImageShow{
+               store: &ImageStore{dir: t.TempDir()},
+               bot:   bot,
+       }
+
+       for _, name := range []string{"../image.png", "unknown", "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa.png"} {
+               _, err := show.Show(t.Context(), name, "Do not send this.")
+               require.Error(t, err)
+       }
+
+       require.Empty(t, telegram.Calls())
+
+       box := &Toolbox{}
+       require.Equal(t, `error: unknown tool "image_show"`, box.run(t.Context(), "image_show", `{"image":"unknown","caption":""}`))
+}
+
+func TestImageShowFallsBackOnlyWhenTelegramRefusesPhoto(t *testing.T) {
+       testCases := []struct {
+               name    string
+               status  int
+               methods []string
+               failure bool
+       }{
+               {
+                       name:    "refused photo sends document",
+                       status:  http.StatusBadRequest,
+                       methods: []string{"sendPhoto", "sendDocument"},
+               },
+               {
+                       name:    "uncertain server error does not send twice",
+                       status:  http.StatusBadGateway,
+                       methods: []string{"sendPhoto"},
+                       failure: true,
+               },
+       }
+       for _, tc := range testCases {
+               t.Run(tc.name, func(t *testing.T) {
+                       telegram := newTelegramFake(t)
+                       telegram.fail = func(call TelegramFakeCall) (int, string) {
+                               if call.method == "sendPhoto" {
+                                       return tc.status, "photo refused"
+                               }
+
+                               return 0, ""
+                       }
+                       store := &ImageStore{dir: t.TempDir()}
+                       picture := imageTestPNG(t, 3, 3)
+                       name, err := store.Save(picture)
+                       require.NoError(t, err)
+
+                       bot, _ := telegramStartTestBot(t, telegram)
+                       show := &ImageShow{
+                               store: store,
+                               bot:   bot,
+                       }
+                       _, err = show.Show(t.Context(), name, "Caption")
+                       if tc.failure {
+                               require.Error(t, err)
+                       } else {
+                               require.NoError(t, err)
+                       }
+
+                       var methods []string
+
+                       for _, call := range telegram.Calls() {
+                               methods = append(methods, call.method)
+                               if call.method == "sendDocument" {
+                                       require.Equal(t, picture, call.params["document"])
+                                       require.Equal(t, name, call.params["document_name"])
+                                       require.Equal(t, "Caption", call.params["caption"])
+                               }
+                       }
+
+                       require.Equal(t, tc.methods, methods)
+               })
+       }
+}
+
+func TestAgentShowsImageInTerminalWithoutTelegram(t *testing.T) {
+       server := newModelFake(t)
+       test := newTestTurnLoop(t, server)
+       name, err := test.agent.Images().Save(imageTestPNG(t, 2, 2))
+       require.NoError(t, err)
+
+       output := test.loop.tools.run(t.Context(), "image_show", `{"image":"`+name+`","caption":"Here it is."}`)
+       require.Equal(t, imageText([]string{name}, "Here it is.")+"\n"+filepath.Join(test.agent.Images().dir, name), output)
+       require.Contains(t, terminalEventText(Event{
+               Kind: "echo",
+               Text: output,
+       }), filepath.Join(test.agent.Images().dir, name))
+}

diff --git a/prompt.go b/prompt.go
index df6f17d4a0c898d6ee2a0a448a916d85c0caf3c6..5847ee82742d07b97862db2ac07ca661ed7236c2 100644 (file)
--- a/prompt.go
+++ b/prompt.go
@@ -274,6 +274,15 @@ func promptTools() []ModelTool {
                        "message": promptStringSchema("The message."),
                }),
                Strict: true,
                        "message": promptStringSchema("The message."),
                }),
                Strict: true,

+       }, ModelTool{
+               Type:        "function",
+               Name:        "image_show",
+               Description: "Show a stored image to the user, only when the user asks to see it. Give its stored filename, not a path.",
+               Parameters: promptObjectSchema(map[string]any{
+                       "image":   promptStringSchema("The stored image filename from an [image FILE] line."),
+                       "caption": promptStringSchema("The caption, or an empty string."),
+               }),
+               Strict: true,

        }, ModelTool{
                Type:        "function",
                Name:        "whatsapp_find",
        }, ModelTool{
                Type:        "function",
                Name:        "whatsapp_find",

diff --git a/prompt_test.go b/prompt_test.go
index e9e87f347b9f2676f5ba5cc8ba46f7c0dd83352f..06fea49cf84315d734120c5d23363113b70fa4ed 100644 (file)
--- a/prompt_test.go
+++ b/prompt_test.go
@@ -129,7 +129,7 @@ func TestProbePromptKeepsTheInstructionsPathStable(t *testing.T) {
 
 func TestAgentToolsAreStrictAndStable(t *testing.T) {
        tools := promptTools()
 
 func TestAgentToolsAreStrictAndStable(t *testing.T) {
        tools := promptTools()

-       require.Equal(t, []string{"zoom", "date", "shell", "read", "write", "edit", "spawn", "agent_create", "tell", "whatsapp_find", "whatsapp_send", "route", "unroute", "routes", "schedule_set", "schedule_list", "schedule_remove"}, promptToolNames(tools))
+       require.Equal(t, []string{"zoom", "date", "shell", "read", "write", "edit", "spawn", "agent_create", "tell", "image_show", "whatsapp_find", "whatsapp_send", "route", "unroute", "routes", "schedule_set", "schedule_list", "schedule_remove"}, promptToolNames(tools))

        require.Equal(t, []string{"zoom", "date", "shell", "read", "write", "edit"}, promptToolNames(promptSubagentTools()))
 
        for _, tool := range tools {
        require.Equal(t, []string{"zoom", "date", "shell", "read", "write", "edit"}, promptToolNames(promptSubagentTools()))
 
        for _, tool := range tools {

diff --git a/telegram_api.go b/telegram_api.go
index 8f78387d096b3215de9a0246ec7213f338d1f601..508010e409e60e84ab089860a528c312e4251169 100644 (file)
--- a/telegram_api.go
+++ b/telegram_api.go
@@ -7,9 +7,11 @@ import (
        "errors"
        "fmt"
        "io"
        "errors"
        "fmt"
        "io"

+       "mime/multipart"

        "net/http"
        "net/url"
        "path"
        "net/http"
        "net/url"
        "path"

+       "strconv"

        "strings"
        "time"
 )
        "strings"
        "time"
 )


@@ -196,6 +198,64 @@ func (b *TelegramAPI) SendRichMessage(ctx context.Context, chat int64, markdown
        }, nil)
 }
 
        }, nil)
 }
 

+// SendImage uploads an image to the owner as a photo, or as a document when the
+// Bot API refuses the photo. Network failures do not trigger a second send.
+func (b *TelegramAPI) SendImage(ctx context.Context, chat int64, name string, data []byte, caption string) error {
+       err := b.sendImage(ctx, "sendPhoto", "photo", chat, name, data, caption)
+       if telegramRejected(err) {
+               err = b.sendImage(ctx, "sendDocument", "document", chat, name, data, caption)
+       }
+
+       return err
+}
+
+func (b *TelegramAPI) sendImage(ctx context.Context, method, field string, chat int64, name string, data []byte, caption string) error {
+       ctx, cancel := context.WithTimeout(ctx, telegramSendTimeout)
+       defer cancel()
+
+       var body bytes.Buffer
+
+       writer := multipart.NewWriter(&body)
+       err := writer.WriteField("chat_id", strconv.FormatInt(chat, 10))
+       if err != nil {
+               return fmt.Errorf("write image chat: %w", err)
+       }
+
+       err = writer.WriteField("caption", caption)
+       if err != nil {
+               return fmt.Errorf("write image caption: %w", err)
+       }
+
+       part, err := writer.CreateFormFile(field, name)
+       if err != nil {
+               return fmt.Errorf("make image part: %w", err)
+       }
+
+       _, err = part.Write(data)
+       if err != nil {
+               return fmt.Errorf("write image part: %w", err)
+       }
+
+       err = writer.Close()
+       if err != nil {
+               return fmt.Errorf("close image form: %w", err)
+       }
+
+       request, err := http.NewRequestWithContext(ctx, http.MethodPost, b.base+"/bot"+b.token+"/"+method, &body)
+       if err != nil {
+               return fmt.Errorf("make image upload: %s", b.redact(err))
+       }
+
+       request.Header.Set("Content-Type", writer.FormDataContentType())
+
+       err = b.answer(request, telegramSendTimeout, nil)
+       if err != nil {
+               return fmt.Errorf("upload image: %w", err)
+       }
+
+       return nil
+}
+

 // SendTyping shows the typing action in chat for about five seconds. A call
 // that takes longer than telegramTypingTimeout fails with a
 // TelegramNetworkError.
 // SendTyping shows the typing action in chat for about five seconds. A call
 // that takes longer than telegramTypingTimeout fails with a
 // TelegramNetworkError.


@@ -226,6 +286,11 @@ func (b *TelegramAPI) call(ctx context.Context, timeout time.Duration, method st
 
        request.Header.Set("Content-Type", "application/json")
 
 
        request.Header.Set("Content-Type", "application/json")
 

+       return b.answer(request, timeout, result)
+}
+
+// answer reads the bounded Bot API response without keeping its request URL.
+func (b *TelegramAPI) answer(request *http.Request, timeout time.Duration, result any) error {

        response, err := b.http.Do(request)
        if err != nil {
                return b.networkError("no answer", err, timeout)
        response, err := b.http.Do(request)
        if err != nil {
                return b.networkError("no answer", err, timeout)

diff --git a/telegram_fake_helper_test.go b/telegram_fake_helper_test.go
index f6ccd8425d41e36372fc51fb8e8a5afc7f2b2216..b71eee038ebf084da3481bd63c3643cda8600c0d 100644 (file)
--- a/telegram_fake_helper_test.go
+++ b/telegram_fake_helper_test.go
@@ -2,8 +2,10 @@ package main
 
 import (
        "encoding/json"
 
 import (
        "encoding/json"

+       "errors"

        "fmt"
        "io"
        "fmt"
        "io"

+       "mime/multipart"

        "net/http"
        "net/http/httptest"
        "strings"
        "net/http"
        "net/http/httptest"
        "strings"


@@ -139,7 +141,38 @@ func (f *TelegramFake) serve(w http.ResponseWriter, r *http.Request) {
 
        var params map[string]any
 
 
        var params map[string]any
 

-       err := json.NewDecoder(r.Body).Decode(&params)
+       var err error
+
+       if strings.HasPrefix(r.Header.Get("Content-Type"), "multipart/form-data;") {
+               err = r.ParseMultipartForm(imageMaxBytes + (1 << 20))
+               if err == nil {
+                       params = make(map[string]any)
+                       for key, values := range r.MultipartForm.Value {
+                               params[key] = values[0]
+                       }
+
+                       for key, files := range r.MultipartForm.File {
+                               var file multipart.File
+
+                               file, err = files[0].Open()
+                               if err != nil {
+                                       break
+                               }
+
+                               params[key], err = io.ReadAll(file)
+                               err = errors.Join(err, file.Close())
+                               params[key+"_name"] = files[0].Filename
+                               if err != nil {
+                                       break
+                               }
+                       }
+
+                       err = errors.Join(err, r.MultipartForm.RemoveAll())
+               }
+       } else {
+               err = json.NewDecoder(r.Body).Decode(&params)
+       }
+

        if err != nil {
                http.Error(w, err.Error(), http.StatusBadRequest)
 
        if err != nil {
                http.Error(w, err.Error(), http.StatusBadRequest)
 

diff --git a/tools.go b/tools.go
index bbd34cf67a7c82389e35aca4944bab7e33024fff..9f2905acb8b32ef61a6c96dbc3cc4365d086d4aa 100644 (file)
--- a/tools.go
+++ b/tools.go
@@ -54,8 +54,8 @@ type ToolChat interface {
 // spawn and tell, the WhatsApp link for the WhatsApp tools, and the schedules
 // for the schedule tools. Each permanent agent has the parent-child queue. Only
 // the root has child creation and source routes. A child has neither of them
 // spawn and tell, the WhatsApp link for the WhatsApp tools, and the schedules
 // for the schedule tools. Each permanent agent has the parent-child queue. Only
 // the root has child creation and source routes. A child has neither of them

-// nor a WhatsApp link. The toolbox of a subagent has none of them, so these are
-// unknown tools there.
+// nor a WhatsApp link. Only the root can show images to the user. The toolbox
+// of a subagent has none of them, so these are unknown tools there.

 type Toolbox struct {
        log       ToolLog
        chat      ToolChat
 type Toolbox struct {
        log       ToolLog
        chat      ToolChat


@@ -69,6 +69,7 @@ type Toolbox struct {
        children  *DaemonChildren
        routes    *Routes
        images    *ImageStore
        children  *DaemonChildren
        routes    *Routes
        images    *ImageStore

+       imageShow *ImageShow

 }
 
 // Call logs the call of the tool name with its JSON arguments, makes the log
 }
 
 // Call logs the call of the tool name with its JSON arguments, makes the log


@@ -231,6 +232,16 @@ func (b *Toolbox) run(ctx context.Context, name, arguments string) string {
                case err == nil:
                        output, err = b.runs.Tell(ctx, args.Name, args.Message)
                }
                case err == nil:
                        output, err = b.runs.Tell(ctx, args.Name, args.Message)
                }

+       case "image_show":
+               var args ToolImageShowArguments
+
+               err = toolDecodeArguments(arguments, &args)
+               switch {
+               case b.imageShow == nil:
+                       err = fmt.Errorf("unknown tool %q", name)
+               case err == nil:
+                       output, err = b.imageShow.Show(ctx, args.Image, args.Caption)
+               }

        case "whatsapp_find":
                var args ToolWhatsAppFindArguments
 
        case "whatsapp_find":
                var args ToolWhatsAppFindArguments
 


@@ -549,6 +560,12 @@ type ToolTellArguments struct {
        Message string `json:"message"`
 }
 
        Message string `json:"message"`
 }
 

+// ToolImageShowArguments are the arguments of image_show.
+type ToolImageShowArguments struct {
+       Image   string `json:"image"`
+       Caption string `json:"caption"`
+}
+

 // ToolWhatsAppFindArguments are the arguments of whatsapp_find.
 type ToolWhatsAppFindArguments struct {
        Name string `json:"name"`
 // ToolWhatsAppFindArguments are the arguments of whatsapp_find.
 type ToolWhatsAppFindArguments struct {
        Name string `json:"name"`

diff --git a/turn.go b/turn.go
index 2c1d9c22d11615aebaa14baa6f424812cbf3609c..9e6a1a7b3a392137bda2cbf95323dc456c4580f3 100644 (file)
--- a/turn.go
+++ b/turn.go
@@ -63,6 +63,10 @@ func newTurnLoop(agent *Agent, memory *Memory, subagents *Subagents, whatsapp *W
                images:    agent.Images(),
        }
 
                images:    agent.Images(),
        }
 

+       if agent.config.Parent == "" {
+               l.tools.imageShow = &ImageShow{store: agent.Images()}
+       }
+

        return l, nil
 }
 
        return l, nil
 }