From: Anderson A. Giacomolli Date: Tue, 10 Nov 2026 18:59:13 +0000 (-0300) Subject: Show stored images to the user X-Git-Url: https://git.giacomolli.uk/onechat/commitdiff_plain/3cdbd017da9ae7a7045f4272f16d4b262fd01c79?ds=inline;hp=add3cf6416e33c24331fd1e90e99284f44d2aae2 Show stored images to the user The root can show a stored image as a Telegram photo and return its name and path to terminal clients. A refused photo goes as a document, while an uncertain send does not cause a second send. Children and subagents have no image output tool. --- diff --git a/agent.go b/agent.go index 97d6685..b86880e 100644 --- a/agent.go +++ b/agent.go @@ -117,12 +117,12 @@ func (a *Agent) Prompt() (string, error) { } // Tools returns the tools shared by the turns and compactions of this agent. A -// child has no WhatsApp, child creation, or route tools. +// child has no image output, WhatsApp, child creation, or route tools. func (a *Agent) Tools() []ModelTool { tools := promptTools() if a.config.Parent != "" { tools = slices.DeleteFunc(tools, func(tool ModelTool) bool { - return strings.HasPrefix(tool.Name, "whatsapp_") || tool.Name == "agent_create" || tool.Name == "route" || tool.Name == "unroute" || tool.Name == "routes" + return strings.HasPrefix(tool.Name, "whatsapp_") || tool.Name == "image_show" || tool.Name == "agent_create" || tool.Name == "route" || tool.Name == "unroute" || tool.Name == "routes" }) } diff --git a/daemon.go b/daemon.go index 137cc6f..5c94a4f 100644 --- a/daemon.go +++ b/daemon.go @@ -353,6 +353,10 @@ func daemonServe(ctx context.Context, listener net.Listener, bot *TelegramBot, w return errors.Join(err, listener.Close()) } + if bot != nil { + l.tools.imageShow.bot = bot + } + loops := []*TurnLoop{l} for _, child := range children { loop, err := child.Loop(nil, func(Event) { diff --git a/daemon_children_test.go b/daemon_children_test.go index 16f326f..b9e212f 100644 --- a/daemon_children_test.go +++ b/daemon_children_test.go @@ -335,6 +335,8 @@ func TestChildToolsExcludeCreationAndWhatsApp(t *testing.T) { names := promptToolNames(child.agent.Tools()) require.NotContains(t, names, "agent_create") + require.NotContains(t, names, "image_show") + require.Equal(t, `error: unknown tool "image_show"`, child.tools.run(t.Context(), "image_show", `{"image":"unknown","caption":""}`)) require.NotContains(t, names, "whatsapp_send") require.NotContains(t, names, "whatsapp_find") require.Contains(t, names, "spawn") diff --git a/image_show.go b/image_show.go new file mode 100644 index 0000000..80204c1 --- /dev/null +++ b/image_show.go @@ -0,0 +1,37 @@ +package main + +import ( + "context" + "fmt" + "path/filepath" +) + +// ImageShow shows a stored image to the root's user. A configured bot sends it +// to its owner, and the tool result gives terminal clients its name and path. +type ImageShow struct { + store *ImageStore + bot *TelegramBot +} + +// Show checks the stored name and bytes before any output. A refused Telegram +// photo goes as a document. An uncertain send is not repeated. +func (s *ImageShow) Show(ctx context.Context, name, caption string) (string, error) { + data, err := s.store.Read(name) + if err != nil { + return "", fmt.Errorf("read image to show: %w", err) + } + + path, err := filepath.Abs(filepath.Join(s.store.dir, name)) + if err != nil { + return "", fmt.Errorf("find image path: %w", err) + } + + if s.bot != nil { + err = s.bot.api.SendImage(ctx, s.bot.owner, name, data, caption) + if err != nil { + return "", fmt.Errorf("show Telegram image: %w", err) + } + } + + return imageText([]string{name}, caption) + "\n" + path, nil +} diff --git a/image_show_test.go b/image_show_test.go new file mode 100644 index 0000000..ed475fe --- /dev/null +++ b/image_show_test.go @@ -0,0 +1,136 @@ +package main + +import ( + "encoding/json" + "net/http" + "path/filepath" + "testing" + + "github.com/stretchr/testify/require" +) + +func TestImageShowUsesStoredImageForTelegramAndTerminal(t *testing.T) { + picture := imageTestPNG(t, 5, 4) + store := &ImageStore{dir: filepath.Join(t.TempDir(), imageDirName)} + name, err := store.Save(picture) + require.NoError(t, err) + + telegram := newTelegramFake(t) + bot, logs := telegramStartTestBot(t, telegram) + box := &Toolbox{imageShow: &ImageShow{ + store: store, + bot: bot, + }} + args, err := json.Marshal(ToolImageShowArguments{ + Image: name, + Caption: "A red point.", + }) + require.NoError(t, err) + + output := box.run(t.Context(), "image_show", string(args)) + require.Equal(t, imageText([]string{name}, "A red point.")+"\n"+filepath.Join(store.dir, name), output) + + calls := telegram.Calls() + require.Len(t, calls, 1) + require.Equal(t, "sendPhoto", calls[0].method) + require.Equal(t, "4242", calls[0].params["chat_id"]) + require.Equal(t, "A red point.", calls[0].params["caption"]) + require.Equal(t, name, calls[0].params["photo_name"]) + require.Equal(t, picture, calls[0].params["photo"]) + require.Empty(t, logs.String()) +} + +func TestImageShowRefusesUnknownNamesBeforeSending(t *testing.T) { + telegram := newTelegramFake(t) + bot, _ := telegramStartTestBot(t, telegram) + show := &ImageShow{ + store: &ImageStore{dir: t.TempDir()}, + bot: bot, + } + + for _, name := range []string{"../image.png", "unknown", "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa.png"} { + _, err := show.Show(t.Context(), name, "Do not send this.") + require.Error(t, err) + } + + require.Empty(t, telegram.Calls()) + + box := &Toolbox{} + require.Equal(t, `error: unknown tool "image_show"`, box.run(t.Context(), "image_show", `{"image":"unknown","caption":""}`)) +} + +func TestImageShowFallsBackOnlyWhenTelegramRefusesPhoto(t *testing.T) { + testCases := []struct { + name string + status int + methods []string + failure bool + }{ + { + name: "refused photo sends document", + status: http.StatusBadRequest, + methods: []string{"sendPhoto", "sendDocument"}, + }, + { + name: "uncertain server error does not send twice", + status: http.StatusBadGateway, + methods: []string{"sendPhoto"}, + failure: true, + }, + } + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + telegram := newTelegramFake(t) + telegram.fail = func(call TelegramFakeCall) (int, string) { + if call.method == "sendPhoto" { + return tc.status, "photo refused" + } + + return 0, "" + } + store := &ImageStore{dir: t.TempDir()} + picture := imageTestPNG(t, 3, 3) + name, err := store.Save(picture) + require.NoError(t, err) + + bot, _ := telegramStartTestBot(t, telegram) + show := &ImageShow{ + store: store, + bot: bot, + } + _, err = show.Show(t.Context(), name, "Caption") + if tc.failure { + require.Error(t, err) + } else { + require.NoError(t, err) + } + + var methods []string + + for _, call := range telegram.Calls() { + methods = append(methods, call.method) + if call.method == "sendDocument" { + require.Equal(t, picture, call.params["document"]) + require.Equal(t, name, call.params["document_name"]) + require.Equal(t, "Caption", call.params["caption"]) + } + } + + require.Equal(t, tc.methods, methods) + }) + } +} + +func TestAgentShowsImageInTerminalWithoutTelegram(t *testing.T) { + server := newModelFake(t) + test := newTestTurnLoop(t, server) + name, err := test.agent.Images().Save(imageTestPNG(t, 2, 2)) + require.NoError(t, err) + + output := test.loop.tools.run(t.Context(), "image_show", `{"image":"`+name+`","caption":"Here it is."}`) + require.Equal(t, imageText([]string{name}, "Here it is.")+"\n"+filepath.Join(test.agent.Images().dir, name), output) + require.Contains(t, terminalEventText(Event{ + Kind: "echo", + Text: output, + }), filepath.Join(test.agent.Images().dir, name)) +} diff --git a/prompt.go b/prompt.go index df6f17d..5847ee8 100644 --- a/prompt.go +++ b/prompt.go @@ -274,6 +274,15 @@ func promptTools() []ModelTool { "message": promptStringSchema("The message."), }), Strict: true, + }, ModelTool{ + Type: "function", + Name: "image_show", + Description: "Show a stored image to the user, only when the user asks to see it. Give its stored filename, not a path.", + Parameters: promptObjectSchema(map[string]any{ + "image": promptStringSchema("The stored image filename from an [image FILE] line."), + "caption": promptStringSchema("The caption, or an empty string."), + }), + Strict: true, }, ModelTool{ Type: "function", Name: "whatsapp_find", diff --git a/prompt_test.go b/prompt_test.go index e9e87f3..06fea49 100644 --- a/prompt_test.go +++ b/prompt_test.go @@ -129,7 +129,7 @@ func TestProbePromptKeepsTheInstructionsPathStable(t *testing.T) { func TestAgentToolsAreStrictAndStable(t *testing.T) { tools := promptTools() - require.Equal(t, []string{"zoom", "date", "shell", "read", "write", "edit", "spawn", "agent_create", "tell", "whatsapp_find", "whatsapp_send", "route", "unroute", "routes", "schedule_set", "schedule_list", "schedule_remove"}, promptToolNames(tools)) + require.Equal(t, []string{"zoom", "date", "shell", "read", "write", "edit", "spawn", "agent_create", "tell", "image_show", "whatsapp_find", "whatsapp_send", "route", "unroute", "routes", "schedule_set", "schedule_list", "schedule_remove"}, promptToolNames(tools)) require.Equal(t, []string{"zoom", "date", "shell", "read", "write", "edit"}, promptToolNames(promptSubagentTools())) for _, tool := range tools { diff --git a/telegram_api.go b/telegram_api.go index 8f78387..508010e 100644 --- a/telegram_api.go +++ b/telegram_api.go @@ -7,9 +7,11 @@ import ( "errors" "fmt" "io" + "mime/multipart" "net/http" "net/url" "path" + "strconv" "strings" "time" ) @@ -196,6 +198,64 @@ func (b *TelegramAPI) SendRichMessage(ctx context.Context, chat int64, markdown }, nil) } +// SendImage uploads an image to the owner as a photo, or as a document when the +// Bot API refuses the photo. Network failures do not trigger a second send. +func (b *TelegramAPI) SendImage(ctx context.Context, chat int64, name string, data []byte, caption string) error { + err := b.sendImage(ctx, "sendPhoto", "photo", chat, name, data, caption) + if telegramRejected(err) { + err = b.sendImage(ctx, "sendDocument", "document", chat, name, data, caption) + } + + return err +} + +func (b *TelegramAPI) sendImage(ctx context.Context, method, field string, chat int64, name string, data []byte, caption string) error { + ctx, cancel := context.WithTimeout(ctx, telegramSendTimeout) + defer cancel() + + var body bytes.Buffer + + writer := multipart.NewWriter(&body) + err := writer.WriteField("chat_id", strconv.FormatInt(chat, 10)) + if err != nil { + return fmt.Errorf("write image chat: %w", err) + } + + err = writer.WriteField("caption", caption) + if err != nil { + return fmt.Errorf("write image caption: %w", err) + } + + part, err := writer.CreateFormFile(field, name) + if err != nil { + return fmt.Errorf("make image part: %w", err) + } + + _, err = part.Write(data) + if err != nil { + return fmt.Errorf("write image part: %w", err) + } + + err = writer.Close() + if err != nil { + return fmt.Errorf("close image form: %w", err) + } + + request, err := http.NewRequestWithContext(ctx, http.MethodPost, b.base+"/bot"+b.token+"/"+method, &body) + if err != nil { + return fmt.Errorf("make image upload: %s", b.redact(err)) + } + + request.Header.Set("Content-Type", writer.FormDataContentType()) + + err = b.answer(request, telegramSendTimeout, nil) + if err != nil { + return fmt.Errorf("upload image: %w", err) + } + + return nil +} + // SendTyping shows the typing action in chat for about five seconds. A call // that takes longer than telegramTypingTimeout fails with a // TelegramNetworkError. @@ -226,6 +286,11 @@ func (b *TelegramAPI) call(ctx context.Context, timeout time.Duration, method st request.Header.Set("Content-Type", "application/json") + return b.answer(request, timeout, result) +} + +// answer reads the bounded Bot API response without keeping its request URL. +func (b *TelegramAPI) answer(request *http.Request, timeout time.Duration, result any) error { response, err := b.http.Do(request) if err != nil { return b.networkError("no answer", err, timeout) diff --git a/telegram_fake_helper_test.go b/telegram_fake_helper_test.go index f6ccd84..b71eee0 100644 --- a/telegram_fake_helper_test.go +++ b/telegram_fake_helper_test.go @@ -2,8 +2,10 @@ package main import ( "encoding/json" + "errors" "fmt" "io" + "mime/multipart" "net/http" "net/http/httptest" "strings" @@ -139,7 +141,38 @@ func (f *TelegramFake) serve(w http.ResponseWriter, r *http.Request) { var params map[string]any - err := json.NewDecoder(r.Body).Decode(¶ms) + var err error + + if strings.HasPrefix(r.Header.Get("Content-Type"), "multipart/form-data;") { + err = r.ParseMultipartForm(imageMaxBytes + (1 << 20)) + if err == nil { + params = make(map[string]any) + for key, values := range r.MultipartForm.Value { + params[key] = values[0] + } + + for key, files := range r.MultipartForm.File { + var file multipart.File + + file, err = files[0].Open() + if err != nil { + break + } + + params[key], err = io.ReadAll(file) + err = errors.Join(err, file.Close()) + params[key+"_name"] = files[0].Filename + if err != nil { + break + } + } + + err = errors.Join(err, r.MultipartForm.RemoveAll()) + } + } else { + err = json.NewDecoder(r.Body).Decode(¶ms) + } + if err != nil { http.Error(w, err.Error(), http.StatusBadRequest) diff --git a/tools.go b/tools.go index bbd34cf..9f2905a 100644 --- a/tools.go +++ b/tools.go @@ -54,8 +54,8 @@ type ToolChat interface { // spawn and tell, the WhatsApp link for the WhatsApp tools, and the schedules // for the schedule tools. Each permanent agent has the parent-child queue. Only // the root has child creation and source routes. A child has neither of them -// nor a WhatsApp link. The toolbox of a subagent has none of them, so these are -// unknown tools there. +// nor a WhatsApp link. Only the root can show images to the user. The toolbox +// of a subagent has none of them, so these are unknown tools there. type Toolbox struct { log ToolLog chat ToolChat @@ -69,6 +69,7 @@ type Toolbox struct { children *DaemonChildren routes *Routes images *ImageStore + imageShow *ImageShow } // Call logs the call of the tool name with its JSON arguments, makes the log @@ -231,6 +232,16 @@ func (b *Toolbox) run(ctx context.Context, name, arguments string) string { case err == nil: output, err = b.runs.Tell(ctx, args.Name, args.Message) } + case "image_show": + var args ToolImageShowArguments + + err = toolDecodeArguments(arguments, &args) + switch { + case b.imageShow == nil: + err = fmt.Errorf("unknown tool %q", name) + case err == nil: + output, err = b.imageShow.Show(ctx, args.Image, args.Caption) + } case "whatsapp_find": var args ToolWhatsAppFindArguments @@ -549,6 +560,12 @@ type ToolTellArguments struct { Message string `json:"message"` } +// ToolImageShowArguments are the arguments of image_show. +type ToolImageShowArguments struct { + Image string `json:"image"` + Caption string `json:"caption"` +} + // ToolWhatsAppFindArguments are the arguments of whatsapp_find. type ToolWhatsAppFindArguments struct { Name string `json:"name"` diff --git a/turn.go b/turn.go index 2c1d9c2..9e6a1a7 100644 --- a/turn.go +++ b/turn.go @@ -63,6 +63,10 @@ func newTurnLoop(agent *Agent, memory *Memory, subagents *Subagents, whatsapp *W images: agent.Images(), } + if agent.config.Parent == "" { + l.tools.imageShow = &ImageShow{store: agent.Images()} + } + return l, nil }