}
// Tools returns the tools shared by the turns and compactions of this agent. A
-// child has no WhatsApp, child creation, or route tools.
+// child has no image output, WhatsApp, child creation, or route tools.
func (a *Agent) Tools() []ModelTool {
tools := promptTools()
if a.config.Parent != "" {
tools = slices.DeleteFunc(tools, func(tool ModelTool) bool {
- return strings.HasPrefix(tool.Name, "whatsapp_") || tool.Name == "agent_create" || tool.Name == "route" || tool.Name == "unroute" || tool.Name == "routes"
+ return strings.HasPrefix(tool.Name, "whatsapp_") || tool.Name == "image_show" || tool.Name == "agent_create" || tool.Name == "route" || tool.Name == "unroute" || tool.Name == "routes"
})
}
return errors.Join(err, listener.Close())
}
+ if bot != nil {
+ l.tools.imageShow.bot = bot
+ }
+
loops := []*TurnLoop{l}
for _, child := range children {
loop, err := child.Loop(nil, func(Event) {
names := promptToolNames(child.agent.Tools())
require.NotContains(t, names, "agent_create")
+ require.NotContains(t, names, "image_show")
+ require.Equal(t, `error: unknown tool "image_show"`, child.tools.run(t.Context(), "image_show", `{"image":"unknown","caption":""}`))
require.NotContains(t, names, "whatsapp_send")
require.NotContains(t, names, "whatsapp_find")
require.Contains(t, names, "spawn")
--- /dev/null
+package main
+
+import (
+ "context"
+ "fmt"
+ "path/filepath"
+)
+
+// ImageShow shows a stored image to the root's user. A configured bot sends it
+// to its owner, and the tool result gives terminal clients its name and path.
+type ImageShow struct {
+ store *ImageStore
+ bot *TelegramBot
+}
+
+// Show checks the stored name and bytes before any output. A refused Telegram
+// photo goes as a document. An uncertain send is not repeated.
+func (s *ImageShow) Show(ctx context.Context, name, caption string) (string, error) {
+ data, err := s.store.Read(name)
+ if err != nil {
+ return "", fmt.Errorf("read image to show: %w", err)
+ }
+
+ path, err := filepath.Abs(filepath.Join(s.store.dir, name))
+ if err != nil {
+ return "", fmt.Errorf("find image path: %w", err)
+ }
+
+ if s.bot != nil {
+ err = s.bot.api.SendImage(ctx, s.bot.owner, name, data, caption)
+ if err != nil {
+ return "", fmt.Errorf("show Telegram image: %w", err)
+ }
+ }
+
+ return imageText([]string{name}, caption) + "\n" + path, nil
+}
--- /dev/null
+package main
+
+import (
+ "encoding/json"
+ "net/http"
+ "path/filepath"
+ "testing"
+
+ "github.com/stretchr/testify/require"
+)
+
+func TestImageShowUsesStoredImageForTelegramAndTerminal(t *testing.T) {
+ picture := imageTestPNG(t, 5, 4)
+ store := &ImageStore{dir: filepath.Join(t.TempDir(), imageDirName)}
+ name, err := store.Save(picture)
+ require.NoError(t, err)
+
+ telegram := newTelegramFake(t)
+ bot, logs := telegramStartTestBot(t, telegram)
+ box := &Toolbox{imageShow: &ImageShow{
+ store: store,
+ bot: bot,
+ }}
+ args, err := json.Marshal(ToolImageShowArguments{
+ Image: name,
+ Caption: "A red point.",
+ })
+ require.NoError(t, err)
+
+ output := box.run(t.Context(), "image_show", string(args))
+ require.Equal(t, imageText([]string{name}, "A red point.")+"\n"+filepath.Join(store.dir, name), output)
+
+ calls := telegram.Calls()
+ require.Len(t, calls, 1)
+ require.Equal(t, "sendPhoto", calls[0].method)
+ require.Equal(t, "4242", calls[0].params["chat_id"])
+ require.Equal(t, "A red point.", calls[0].params["caption"])
+ require.Equal(t, name, calls[0].params["photo_name"])
+ require.Equal(t, picture, calls[0].params["photo"])
+ require.Empty(t, logs.String())
+}
+
+func TestImageShowRefusesUnknownNamesBeforeSending(t *testing.T) {
+ telegram := newTelegramFake(t)
+ bot, _ := telegramStartTestBot(t, telegram)
+ show := &ImageShow{
+ store: &ImageStore{dir: t.TempDir()},
+ bot: bot,
+ }
+
+ for _, name := range []string{"../image.png", "unknown", "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa.png"} {
+ _, err := show.Show(t.Context(), name, "Do not send this.")
+ require.Error(t, err)
+ }
+
+ require.Empty(t, telegram.Calls())
+
+ box := &Toolbox{}
+ require.Equal(t, `error: unknown tool "image_show"`, box.run(t.Context(), "image_show", `{"image":"unknown","caption":""}`))
+}
+
+func TestImageShowFallsBackOnlyWhenTelegramRefusesPhoto(t *testing.T) {
+ testCases := []struct {
+ name string
+ status int
+ methods []string
+ failure bool
+ }{
+ {
+ name: "refused photo sends document",
+ status: http.StatusBadRequest,
+ methods: []string{"sendPhoto", "sendDocument"},
+ },
+ {
+ name: "uncertain server error does not send twice",
+ status: http.StatusBadGateway,
+ methods: []string{"sendPhoto"},
+ failure: true,
+ },
+ }
+ for _, tc := range testCases {
+ t.Run(tc.name, func(t *testing.T) {
+ telegram := newTelegramFake(t)
+ telegram.fail = func(call TelegramFakeCall) (int, string) {
+ if call.method == "sendPhoto" {
+ return tc.status, "photo refused"
+ }
+
+ return 0, ""
+ }
+ store := &ImageStore{dir: t.TempDir()}
+ picture := imageTestPNG(t, 3, 3)
+ name, err := store.Save(picture)
+ require.NoError(t, err)
+
+ bot, _ := telegramStartTestBot(t, telegram)
+ show := &ImageShow{
+ store: store,
+ bot: bot,
+ }
+ _, err = show.Show(t.Context(), name, "Caption")
+ if tc.failure {
+ require.Error(t, err)
+ } else {
+ require.NoError(t, err)
+ }
+
+ var methods []string
+
+ for _, call := range telegram.Calls() {
+ methods = append(methods, call.method)
+ if call.method == "sendDocument" {
+ require.Equal(t, picture, call.params["document"])
+ require.Equal(t, name, call.params["document_name"])
+ require.Equal(t, "Caption", call.params["caption"])
+ }
+ }
+
+ require.Equal(t, tc.methods, methods)
+ })
+ }
+}
+
+func TestAgentShowsImageInTerminalWithoutTelegram(t *testing.T) {
+ server := newModelFake(t)
+ test := newTestTurnLoop(t, server)
+ name, err := test.agent.Images().Save(imageTestPNG(t, 2, 2))
+ require.NoError(t, err)
+
+ output := test.loop.tools.run(t.Context(), "image_show", `{"image":"`+name+`","caption":"Here it is."}`)
+ require.Equal(t, imageText([]string{name}, "Here it is.")+"\n"+filepath.Join(test.agent.Images().dir, name), output)
+ require.Contains(t, terminalEventText(Event{
+ Kind: "echo",
+ Text: output,
+ }), filepath.Join(test.agent.Images().dir, name))
+}
"message": promptStringSchema("The message."),
}),
Strict: true,
+ }, ModelTool{
+ Type: "function",
+ Name: "image_show",
+ Description: "Show a stored image to the user, only when the user asks to see it. Give its stored filename, not a path.",
+ Parameters: promptObjectSchema(map[string]any{
+ "image": promptStringSchema("The stored image filename from an [image FILE] line."),
+ "caption": promptStringSchema("The caption, or an empty string."),
+ }),
+ Strict: true,
}, ModelTool{
Type: "function",
Name: "whatsapp_find",
func TestAgentToolsAreStrictAndStable(t *testing.T) {
tools := promptTools()
- require.Equal(t, []string{"zoom", "date", "shell", "read", "write", "edit", "spawn", "agent_create", "tell", "whatsapp_find", "whatsapp_send", "route", "unroute", "routes", "schedule_set", "schedule_list", "schedule_remove"}, promptToolNames(tools))
+ require.Equal(t, []string{"zoom", "date", "shell", "read", "write", "edit", "spawn", "agent_create", "tell", "image_show", "whatsapp_find", "whatsapp_send", "route", "unroute", "routes", "schedule_set", "schedule_list", "schedule_remove"}, promptToolNames(tools))
require.Equal(t, []string{"zoom", "date", "shell", "read", "write", "edit"}, promptToolNames(promptSubagentTools()))
for _, tool := range tools {
"errors"
"fmt"
"io"
+ "mime/multipart"
"net/http"
"net/url"
"path"
+ "strconv"
"strings"
"time"
)
}, nil)
}
+// SendImage uploads an image to the owner as a photo, or as a document when the
+// Bot API refuses the photo. Network failures do not trigger a second send.
+func (b *TelegramAPI) SendImage(ctx context.Context, chat int64, name string, data []byte, caption string) error {
+ err := b.sendImage(ctx, "sendPhoto", "photo", chat, name, data, caption)
+ if telegramRejected(err) {
+ err = b.sendImage(ctx, "sendDocument", "document", chat, name, data, caption)
+ }
+
+ return err
+}
+
+func (b *TelegramAPI) sendImage(ctx context.Context, method, field string, chat int64, name string, data []byte, caption string) error {
+ ctx, cancel := context.WithTimeout(ctx, telegramSendTimeout)
+ defer cancel()
+
+ var body bytes.Buffer
+
+ writer := multipart.NewWriter(&body)
+ err := writer.WriteField("chat_id", strconv.FormatInt(chat, 10))
+ if err != nil {
+ return fmt.Errorf("write image chat: %w", err)
+ }
+
+ err = writer.WriteField("caption", caption)
+ if err != nil {
+ return fmt.Errorf("write image caption: %w", err)
+ }
+
+ part, err := writer.CreateFormFile(field, name)
+ if err != nil {
+ return fmt.Errorf("make image part: %w", err)
+ }
+
+ _, err = part.Write(data)
+ if err != nil {
+ return fmt.Errorf("write image part: %w", err)
+ }
+
+ err = writer.Close()
+ if err != nil {
+ return fmt.Errorf("close image form: %w", err)
+ }
+
+ request, err := http.NewRequestWithContext(ctx, http.MethodPost, b.base+"/bot"+b.token+"/"+method, &body)
+ if err != nil {
+ return fmt.Errorf("make image upload: %s", b.redact(err))
+ }
+
+ request.Header.Set("Content-Type", writer.FormDataContentType())
+
+ err = b.answer(request, telegramSendTimeout, nil)
+ if err != nil {
+ return fmt.Errorf("upload image: %w", err)
+ }
+
+ return nil
+}
+
// SendTyping shows the typing action in chat for about five seconds. A call
// that takes longer than telegramTypingTimeout fails with a
// TelegramNetworkError.
request.Header.Set("Content-Type", "application/json")
+ return b.answer(request, timeout, result)
+}
+
+// answer reads the bounded Bot API response without keeping its request URL.
+func (b *TelegramAPI) answer(request *http.Request, timeout time.Duration, result any) error {
response, err := b.http.Do(request)
if err != nil {
return b.networkError("no answer", err, timeout)
import (
"encoding/json"
+ "errors"
"fmt"
"io"
+ "mime/multipart"
"net/http"
"net/http/httptest"
"strings"
var params map[string]any
- err := json.NewDecoder(r.Body).Decode(¶ms)
+ var err error
+
+ if strings.HasPrefix(r.Header.Get("Content-Type"), "multipart/form-data;") {
+ err = r.ParseMultipartForm(imageMaxBytes + (1 << 20))
+ if err == nil {
+ params = make(map[string]any)
+ for key, values := range r.MultipartForm.Value {
+ params[key] = values[0]
+ }
+
+ for key, files := range r.MultipartForm.File {
+ var file multipart.File
+
+ file, err = files[0].Open()
+ if err != nil {
+ break
+ }
+
+ params[key], err = io.ReadAll(file)
+ err = errors.Join(err, file.Close())
+ params[key+"_name"] = files[0].Filename
+ if err != nil {
+ break
+ }
+ }
+
+ err = errors.Join(err, r.MultipartForm.RemoveAll())
+ }
+ } else {
+ err = json.NewDecoder(r.Body).Decode(¶ms)
+ }
+
if err != nil {
http.Error(w, err.Error(), http.StatusBadRequest)
// spawn and tell, the WhatsApp link for the WhatsApp tools, and the schedules
// for the schedule tools. Each permanent agent has the parent-child queue. Only
// the root has child creation and source routes. A child has neither of them
-// nor a WhatsApp link. The toolbox of a subagent has none of them, so these are
-// unknown tools there.
+// nor a WhatsApp link. Only the root can show images to the user. The toolbox
+// of a subagent has none of them, so these are unknown tools there.
type Toolbox struct {
log ToolLog
chat ToolChat
children *DaemonChildren
routes *Routes
images *ImageStore
+ imageShow *ImageShow
}
// Call logs the call of the tool name with its JSON arguments, makes the log
case err == nil:
output, err = b.runs.Tell(ctx, args.Name, args.Message)
}
+ case "image_show":
+ var args ToolImageShowArguments
+
+ err = toolDecodeArguments(arguments, &args)
+ switch {
+ case b.imageShow == nil:
+ err = fmt.Errorf("unknown tool %q", name)
+ case err == nil:
+ output, err = b.imageShow.Show(ctx, args.Image, args.Caption)
+ }
case "whatsapp_find":
var args ToolWhatsAppFindArguments
Message string `json:"message"`
}
+// ToolImageShowArguments are the arguments of image_show.
+type ToolImageShowArguments struct {
+ Image string `json:"image"`
+ Caption string `json:"caption"`
+}
+
// ToolWhatsAppFindArguments are the arguments of whatsapp_find.
type ToolWhatsAppFindArguments struct {
Name string `json:"name"`
images: agent.Images(),
}
+ if agent.config.Parent == "" {
+ l.tools.imageShow = &ImageShow{store: agent.Images()}
+ }
+
return l, nil
}
projects / onechat / commitdiff