WhatsApp sends accept a stored image and use the text as its caption.
The send record includes the image name and precedes the upload, so a
repeated ID cannot upload or send again. Media errors keep safe causes
without URLs or keys.
The root asks for consent as for text sends, and image markers in tell
give children the shared pixels. The user can also ask to see a received
Telegram photo without a separate import.
Images
Send a photo in Telegram, WhatsApp, or with ``/image`` to the agent.
Its memory keeps what the image shows, and it can look at an older
- image again.
+ image again. Ask it to show an image or send one to a WhatsApp chat.
Schedules
Ask for reminders and timed tasks, such as "every weekday at 8, tell
package main
import (
+ "encoding/base64"
"encoding/json"
"net/http"
"path/filepath"
Text: output,
}), filepath.Join(test.agent.Images().dir, name))
}
+
+func TestTellImageMarkerGivesTheChildPixels(t *testing.T) {
+ server := newModelFake(t)
+ test := newDaemonMessageTest(t, t.TempDir(), server)
+ picture := imageTestPNG(t, 4, 2)
+ name, err := test.agents[0].agent.Images().Save(picture)
+ require.NoError(t, err)
+ test.Start(t, 0, 1)
+
+ arguments, err := json.Marshal(ToolTellArguments{
+ Name: "Garden",
+ Message: imageText([]string{name}, "Review this plan."),
+ })
+ require.NoError(t, err)
+ require.Contains(t, test.loops[0].tools.run(t.Context(), "tell", string(arguments)), "Sent.")
+ test.events[1].WaitFor(t, "idle", 1)
+
+ var childCall *ModelFakeCall
+
+ for i := range server.Calls() {
+ call := server.Call(i)
+ if call.body["model"] == "gpt-test-garden" {
+ childCall = &call
+ break
+ }
+ }
+
+ require.NotNil(t, childCall)
+ require.Contains(t, string(childCall.raw), "input_image")
+ require.Contains(t, string(childCall.raw), base64.StdEncoding.EncodeToString(picture))
+ require.Contains(t, test.events[1].Last("user"), "[image "+name+"]")
+ require.NotContains(t, promptToolNames(test.agents[1].agent.Tools()), "image_show")
+}
+
+func TestAgentShowsTelegramReceivedImageToItsOwner(t *testing.T) {
+ picture := imageTestPNG(t, 6, 4)
+ store := &ImageStore{dir: t.TempDir()}
+ name, err := store.Save(picture)
+ require.NoError(t, err)
+
+ telegram := newTelegramFake(t)
+ telegram.Image("owner-photo", picture)
+
+ bot, logs := telegramStartTestBot(t, telegram)
+ server := newModelFake(t)
+ server.reply = func(w http.ResponseWriter, call ModelFakeCall) {
+ if call.number == 0 {
+ arguments, err := json.Marshal(ToolImageShowArguments{
+ Image: name,
+ Caption: "Here is your photo.",
+ })
+ require.NoError(t, err)
+ modelWriteEvents(w, modelCompletedEvent([]any{modelFunctionCallItem("fc_1", "call_1", "image_show", string(arguments))}, 100, 0, 5))
+
+ return
+ }
+
+ modelWriteEvents(w, modelCompletedEvent([]any{modelMessageItem("Shown.")}, 100, 0, 5))
+ }
+ socket := daemonStartTestWithBot(t, server, bot)
+ terminal := terminalStartTestClient(t, socket)
+ terminal.WaitFor(t, "[connected to OneChat]")
+
+ message := telegramOwnerMessage("")
+ message.Caption = "Show this photo."
+ message.Photo = []TelegramPhoto{{
+ FileID: "owner-photo",
+ Width: 6,
+ Height: 4,
+ }}
+ telegram.Add(TelegramUpdate{
+ UpdateID: 1,
+ Message: message,
+ })
+ terminal.WaitFor(t, "Shown.\n[done]")
+ require.Contains(t, terminal.Output(), "[result] [image "+name+"]")
+ require.Contains(t, terminal.Output(), filepath.Join(imageDirName, name))
+
+ var photos []TelegramFakeCall
+
+ for _, call := range telegram.Calls() {
+ if call.method == "sendPhoto" {
+ photos = append(photos, call)
+ }
+ }
+
+ require.Len(t, photos, 1)
+ require.Equal(t, picture, photos[0].params["photo"])
+ require.Equal(t, "4242", photos[0].params["chat_id"])
+ require.Equal(t, "Here is your photo.", photos[0].params["caption"])
+ require.NotContains(t, logs.String(), "Here is your photo.")
+ require.NotContains(t, logs.String(), "Show this photo.")
+ require.NotContains(t, logs.String(), telegramTestToken)
+}
words. Never send a WhatsApp reply drafted by a child without the user's
agreement.
Propose routing a source that talks only about one area to its child.
-Set or remove a route only after the user agrees.`
+Set or remove a route only after the user agrees.
+Use image_show or whatsapp_send with an image only on the user's request.
+Pass an image to a child with an [image FILE] line in tell.`
// promptSubagentText is the system prompt of a subagent, from the subagent
// prompt of the reference design: its head, then the doc of the view, without
}, ModelTool{
Type: "function",
Name: "whatsapp_send",
- Description: "Send a text message to a WhatsApp chat. Send only a message that the user asked you to send, to the chat that the user named. Give a new id for each message: a call with an id that was used gives the answer of the first call and sends nothing.",
+ Description: "Send text or a stored image with text as its caption to a WhatsApp chat. Send only a message that the user asked you to send, to the chat that the user named. Give a new id for each message: a call with an id that was used gives the answer of the first call and sends nothing.",
Parameters: promptObjectSchema(map[string]any{
- "id": promptStringSchema("A new id for this message, such as a short random word."),
- "chat": promptStringSchema("The ID of the chat, from whatsapp_find or from a [WhatsApp] message."),
- "text": promptStringSchema("The text of the message."),
+ "id": promptStringSchema("A new id for this message, such as a short random word."),
+ "chat": promptStringSchema("The ID of the chat, from whatsapp_find or from a [WhatsApp] message."),
+ "text": promptStringSchema("The text, or the caption when sending an image. Can be empty with an image."),
+ "image": promptNullableStringSchema("The stored image filename from an [image FILE] line, or null for text only."),
}),
Strict: true,
}, ModelTool{
require.NotContains(t, child, "Use agent_create")
require.Contains(t, child, "Send nothing to WhatsApp. Draft replies in your report instead")
}
+
+func TestRootPromptKeepsImageConsentAndChildForwardingRules(t *testing.T) {
+ text := prompt("Ada", "/AGENT/instructions.md", "")
+ require.Contains(t, text, "Use image_show or whatsapp_send with an image only on the user's request.")
+ require.Contains(t, text, "Pass an image to a child with an [image FILE] line in tell.")
+
+ child := promptChild("House", "Ada", "Keep house plans.", "/CHILD/instructions.md", "")
+ require.Contains(t, child, "Send nothing to WhatsApp. Draft replies in your report instead")
+ require.NotContains(t, promptToolNames(promptSubagentTools()), "image_show")
+ require.NotContains(t, promptToolNames(promptSubagentTools()), "whatsapp_send")
+}
case b.whatsapp == nil:
err = fmt.Errorf("unknown tool %q", name)
case err == nil:
- output, err = b.whatsapp.Send(ctx, args.ID, args.Chat, args.Text)
+ output, err = b.whatsapp.Send(ctx, args.ID, args.Chat, args.Text, args.Image)
}
case "schedule_set":
var args ToolScheduleSetArguments
// ToolWhatsAppSendArguments are the arguments of whatsapp_send.
type ToolWhatsAppSendArguments struct {
- ID string `json:"id"`
- Chat string `json:"chat"`
- Text string `json:"text"`
+ ID string `json:"id"`
+ Chat string `json:"chat"`
+ Text string `json:"text"`
+ Image string `json:"image"`
}
// ToolScheduleSetArguments are the arguments of schedule_set. Exactly one of At
Logout(ctx context.Context) error
GetJoinedGroups(ctx context.Context) ([]*types.GroupInfo, error)
Download(ctx context.Context, message whatsmeow.DownloadableMessage) ([]byte, error)
+ Upload(ctx context.Context, data []byte, mediaType whatsmeow.MediaType) (whatsmeow.UploadResponse, error)
GenerateMessageID() types.MessageID
SendMessage(ctx context.Context, to types.JID, message *waE2E.Message, extra ...whatsmeow.SendRequestExtra) (whatsmeow.SendResponse, error)
}
return whatsappFind(ctx, w.link(), w.groups, name)
}
-// Send returns the answer of whatsapp_send for the send id of text to chat, as
-// whatsappSend says.
-func (w *WhatsApp) Send(ctx context.Context, id, chat, text string) (string, error) {
- return whatsappSend(ctx, w.link(), w.groups, w.state, w.clock, id, chat, text)
+// Send returns the answer of whatsapp_send for text and an optional stored
+// image to chat, with the durable send id that whatsappSend describes.
+func (w *WhatsApp) Send(ctx context.Context, id, chat, text, image string) (string, error) {
+ return whatsappSend(ctx, w.link(), w.groups, w.state, w.inbox.images, w.clock, id, chat, text, image)
}
func (w *WhatsApp) sessionPath() string {
logoutErr error
groups []*types.GroupInfo
sendErr error
+ uploadErr error
+ uploads [][]byte
+ imageSends []*waE2E.ImageMessage
downloadData []byte
downloadErr error
downloadWait time.Duration
return slices.Clone(f.downloads)
}
+// Upload records the pixels and returns made-up media metadata.
+func (f *WhatsAppFake) Upload(_ context.Context, data []byte, mediaType whatsmeow.MediaType) (whatsmeow.UploadResponse, error) {
+ f.mutex.Lock()
+ defer f.mutex.Unlock()
+
+ if mediaType != whatsmeow.MediaImage {
+ return whatsmeow.UploadResponse{}, fmt.Errorf("unexpected media type %q", mediaType)
+ }
+
+ f.uploads = append(f.uploads, slices.Clone(data))
+ if f.uploadErr != nil {
+ return whatsmeow.UploadResponse{}, f.uploadErr
+ }
+
+ return whatsmeow.UploadResponse{
+ URL: "https://example.com/image",
+ DirectPath: "/image",
+ MediaKey: []byte("made-up key"),
+ FileSHA256: []byte("made-up hash"),
+ FileEncSHA256: []byte("made-up encrypted hash"),
+ FileLength: uint64(len(data)),
+ }, nil
+}
+
+// Uploads returns the pixels passed to Upload.
+func (f *WhatsAppFake) Uploads() [][]byte {
+ f.mutex.Lock()
+ defer f.mutex.Unlock()
+
+ return slices.Clone(f.uploads)
+}
+
+// ImageSends returns the outgoing image messages.
+func (f *WhatsAppFake) ImageSends() []*waE2E.ImageMessage {
+ f.mutex.Lock()
+ defer f.mutex.Unlock()
+
+ return slices.Clone(f.imageSends)
+}
+
// GenerateMessageID returns a new message ID.
func (f *WhatsAppFake) GenerateMessageID() types.MessageID {
f.mutex.Lock()
f.mutex.Lock()
defer f.mutex.Unlock()
+ if message.GetImageMessage() != nil {
+ f.imageSends = append(f.imageSends, message.GetImageMessage())
+ }
+
f.sent = append(f.sent, WhatsAppFakeSend{
to: to.String(),
text: message.GetConversation(),
"context"
"errors"
"fmt"
+ "net/http"
+ "time"
"go.mau.fi/whatsmeow"
"go.mau.fi/whatsmeow/proto/waE2E"
)
+// whatsappImageUploadTimeout limits one image upload to WhatsApp.
+const whatsappImageUploadTimeout = 2 * time.Minute
+
// whatsappImage downloads only supported images and stores them before the
// event can be acknowledged. Both the declared size and the downloaded bytes
// must fit the image store. Decoded bytes determine the stored format.
return errors.New("media download failed")
}
+
+// whatsappImageMessage uploads stored pixels and uses the upload keys only in
+// the outgoing message. The text is the caption, not a separate message.
+func whatsappImageMessage(ctx context.Context, client WhatsAppClient, data []byte, text string) (*waE2E.Message, error) {
+ ctx, cancel := context.WithTimeout(ctx, whatsappImageUploadTimeout)
+ defer cancel()
+
+ upload, err := client.Upload(ctx, data, whatsmeow.MediaImage)
+ if err != nil {
+ return nil, fmt.Errorf("upload image: %w", whatsappImageSendError(err))
+ }
+
+ return &waE2E.Message{ImageMessage: &waE2E.ImageMessage{
+ Caption: new(text),
+ Mimetype: new(http.DetectContentType(data)),
+ URL: &upload.URL,
+ DirectPath: &upload.DirectPath,
+ MediaKey: upload.MediaKey,
+ FileEncSHA256: upload.FileEncSHA256,
+ FileSHA256: upload.FileSHA256,
+ FileLength: &upload.FileLength,
+ }}, nil
+}
+
+// whatsappImageSendError drops arbitrary upload and send errors, which can hold
+// media URLs and keys. Cancellation and deadline causes are safe to keep.
+func whatsappImageSendError(err error) error {
+ switch {
+ case errors.Is(err, context.Canceled):
+ return context.Canceled
+ case errors.Is(err, context.DeadlineExceeded):
+ return context.DeadlineExceeded
+ default:
+ return errors.New("media send failed")
+ }
+}
whatsappFailed = "failed"
)
-// whatsappSend sends text to chat, the ID of a private chat or a group, and
-// returns the answer of the tool. id names the send: a call with an id that has
-// a record gives the answer of the record and sends nothing, and the same id
-// with another chat or text is an error. The record "sending" is durable before
-// the send, so a stop of the daemon during the send gives an answer that the
-// result is unknown, and not a second send. A disconnected WhatsApp sends
-// nothing and makes no record. link gives the device, the client, and the own
-// IDs, groups the names of the groups, and state the send records. The clock
-// gives the times of the records and the time zone of the answer.
-func whatsappSend(ctx context.Context, link WhatsAppLink, groups *WhatsAppGroups, state *WhatsAppState, clock func() time.Time, id, chat, text string) (string, error) {
+// whatsappSend sends text and an optional stored image to chat, the ID of a
+// private chat or a group, and returns the answer of the tool. id names the
+// send: a call with an id that has a record gives the answer of the record and
+// sends nothing, and the same id with another chat, text, or image is an error.
+// The record "sending" is durable before the send, so a stop of the daemon
+// during the send gives an answer that the result is unknown, and not a second
+// send. A disconnected WhatsApp sends nothing and makes no record. link gives
+// the device, the client, and the own IDs, groups the names of the groups, and
+// state the send records. The clock gives the times of the records and the time
+// zone of the answer. Image names are checked before the record is written, and
+// uploads start only after the record is durable. Text-only records keep their
+// text digest.
+func whatsappSend(ctx context.Context, link WhatsAppLink, groups *WhatsAppGroups, state *WhatsAppState, images *ImageStore, clock func() time.Time, id, chat, text, image string) (string, error) {
if !link.linked {
return whatsappNotLinked, nil
}
- jid, err := whatsappSendCheck(id, chat, text, link.own)
+ jid, err := whatsappSendCheck(id, chat, text, image, link.own)
if err != nil {
return "", err
}
digest := sha256.Sum256([]byte(text))
record := WhatsAppSend{
ID: id,
+ Image: image,
Chat: chat,
Digest: hex.EncodeToString(digest[:]),
State: whatsappSending,
held, found := state.Send(id)
switch {
- case found && (held.Chat != record.Chat || held.Digest != record.Digest):
+ case found && (held.Chat != record.Chat || held.Digest != record.Digest || held.Image != record.Image):
return "", fmt.Errorf("the id %s belongs to another message: give a new id for each message", id)
case found && held.State == whatsappSending:
return "The send with this id started, but its result is unknown, because the daemon stopped during it. Ask the user to look at the chat in WhatsApp before you send the message again with a new id.", nil
return "", errors.New("WhatsApp is disconnected, so nothing was sent: try again later")
}
+ var data []byte
+
+ if image != "" {
+ data, err = images.Read(image)
+ if err != nil {
+ return "", fmt.Errorf("read image to send: %w", err)
+ }
+ }
+
record.MessageID = link.client.GenerateMessageID()
err = state.PutSend(record)
if err != nil {
return "", err
}
+ message := &waE2E.Message{Conversation: new(text)}
+ if image != "" {
+ message, err = whatsappImageMessage(ctx, link.client, data, text)
+ }
+
sendCtx, cancel := context.WithTimeout(ctx, whatsappSendTimeout)
defer cancel()
- response, err := link.client.SendMessage(sendCtx, jid, &waE2E.Message{
- Conversation: new(text),
- }, whatsmeow.SendRequestExtra{
- ID: record.MessageID,
- })
+ var response whatsmeow.SendResponse
+
+ if err == nil {
+ response, err = link.client.SendMessage(sendCtx, jid, message, whatsmeow.SendRequestExtra{
+ ID: record.MessageID,
+ })
+ if err != nil && image != "" {
+ err = fmt.Errorf("send image: %w", whatsappImageSendError(err))
+ }
+ }
record.State = whatsappSent
if err != nil {
}
// whatsappSendCheck returns the JID of chat, and an error when id, chat, or
-// text cannot go to a send. The chat is a private chat or a group, and not the
-// chat of the user with itself.
-func whatsappSendCheck(id, chat, text string, own []string) (types.JID, error) {
+// text cannot go to a send. Text can be empty only with an image. The chat is a
+// private chat or a group, and not the chat of the user with itself.
+func whatsappSendCheck(id, chat, text, image string, own []string) (types.JID, error) {
jid, err := types.ParseJID(chat)
switch {
case strings.TrimSpace(id) == "" || len(id) > whatsappSendIDBytes:
return types.EmptyJID, fmt.Errorf("the id must have 1 to %d bytes", whatsappSendIDBytes)
- case strings.TrimSpace(text) == "" || len(text) > whatsappSendBytes:
+ case (strings.TrimSpace(text) == "" && image == "") || len(text) > whatsappSendBytes:
return types.EmptyJID, fmt.Errorf("the text must have 1 to %d bytes", whatsappSendBytes)
case err != nil || jid.User == "" || jid != jid.ToNonAD() || jid.String() != chat:
return types.EmptyJID, fmt.Errorf("%q is not the ID of a WhatsApp chat: find the ID with whatsapp_find", chat)
package main
import (
+ "context"
"crypto/sha256"
"encoding/hex"
"errors"
"path/filepath"
"strings"
"testing"
+ "testing/synctest"
+ "time"
+
+ "go.mau.fi/whatsmeow"
"github.com/stretchr/testify/require"
"go.mau.fi/whatsmeow/types"
daemonStartTestWithWhatsApp(t, newModelFake(t), nil, whatsapp)
whatsappTestConnect(t, fake)
- answer, err := whatsapp.Send(t.Context(), "keys", whatsappTestAna.String(), "The keys are under the mat.")
+ answer, err := whatsapp.Send(t.Context(), "keys", whatsappTestAna.String(), "The keys are under the mat.", "")
require.NoError(t, err)
require.Equal(t, "Sent to Ana Souza at 12:05.", answer)
- answer, err = whatsapp.Send(t.Context(), "keys", whatsappTestAna.String(), "The keys are under the mat.")
+ answer, err = whatsapp.Send(t.Context(), "keys", whatsappTestAna.String(), "The keys are under the mat.", "")
require.NoError(t, err)
require.Equal(t, "Sent to Ana Souza at 12:05. This is the answer of the first call with this id: nothing was sent again.", answer)
- _, err = whatsapp.Send(t.Context(), "keys", whatsappTestAna.String(), "Another text.")
+ _, err = whatsapp.Send(t.Context(), "keys", whatsappTestAna.String(), "Another text.", "")
require.EqualError(t, err, "the id keys belongs to another message: give a new id for each message")
require.Equal(t, []WhatsAppFakeSend{
daemonStartTestWithWhatsApp(t, newModelFake(t), nil, whatsapp)
whatsappTestConnect(t, fake)
- answer, err := whatsapp.Send(t.Context(), "cake", whatsappTestFamily.String(), "I bring the cake.")
+ answer, err := whatsapp.Send(t.Context(), "cake", whatsappTestFamily.String(), "I bring the cake.", "")
require.NoError(t, err)
require.Equal(t, "Sent to Family at 12:05.", answer)
}
whatsappTestConnect(t, fake)
for range 2 {
- answer, err := whatsapp.Send(t.Context(), "keys", whatsappTestAna.String(), "hi")
+ answer, err := whatsapp.Send(t.Context(), "keys", whatsappTestAna.String(), "hi", "")
require.NoError(t, err)
require.True(t, strings.HasPrefix(answer, "The send failed: server error for OWN."), answer)
}
State: whatsappSending,
}))
- answer, err := whatsapp.Send(t.Context(), "keys", whatsappTestAna.String(), "hi")
+ answer, err := whatsapp.Send(t.Context(), "keys", whatsappTestAna.String(), "hi", "")
require.NoError(t, err)
require.Contains(t, answer, "its result is unknown")
require.Empty(t, fake.Sent())
for _, tc := range testCases {
t.Run(tc.name, func(t *testing.T) {
- _, err := whatsapp.Send(t.Context(), tc.id, tc.chat, tc.text)
+ _, err := whatsapp.Send(t.Context(), tc.id, tc.chat, tc.text, "")
require.EqualError(t, err, tc.err)
})
}
whatsappTestConnect(t, fake)
require.True(t, fake.Emit(&events.Disconnected{}))
- _, err := whatsapp.Send(t.Context(), "keys", whatsappTestAna.String(), "hi")
+ _, err := whatsapp.Send(t.Context(), "keys", whatsappTestAna.String(), "hi", "")
require.EqualError(t, err, "WhatsApp is disconnected, so nothing was sent: try again later")
require.True(t, fake.Emit(&events.Connected{}))
- answer, err := whatsapp.Send(t.Context(), "keys", whatsappTestAna.String(), "hi")
+ answer, err := whatsapp.Send(t.Context(), "keys", whatsappTestAna.String(), "hi", "")
require.NoError(t, err)
require.Equal(t, "Sent to +15550002222 at 12:05.", answer)
}
func TestWhatsAppSendWithoutALink(t *testing.T) {
whatsapp, _, _ := whatsappStartTest(t, t.TempDir(), newTestClock())
- answer, err := whatsapp.Send(t.Context(), "keys", whatsappTestAna.String(), "hi")
+ answer, err := whatsapp.Send(t.Context(), "keys", whatsappTestAna.String(), "hi", "")
require.NoError(t, err)
require.Equal(t, whatsappNotLinked, answer)
}
client.WaitFor(t, "[result] Sent to Ana Souza at 12:05.\nSent.\n[done]")
require.Len(t, fake.Sent(), 1)
}
+
+func TestWhatsAppImageSendUploadsOnceAndKeepsCaption(t *testing.T) {
+ whatsapp, fake := whatsappStartTestLinked(t)
+ daemonStartTestWithWhatsApp(t, newModelFake(t), nil, whatsapp)
+ whatsappTestConnect(t, fake)
+
+ picture := imageTestPNG(t, 4, 3)
+ name, err := whatsapp.inbox.images.Save(picture)
+ require.NoError(t, err)
+
+ box := &Toolbox{whatsapp: whatsapp}
+ arguments := `{"id":"photo","chat":"15550002222@s.whatsapp.net","text":"The plan.","image":"` + name + `"}`
+ answer := box.run(t.Context(), "whatsapp_send", arguments)
+ require.Contains(t, answer, "Sent to")
+ require.Contains(t, box.run(t.Context(), "whatsapp_send", arguments), "nothing was sent again")
+ require.Equal(t, [][]byte{picture}, fake.Uploads())
+ require.Len(t, fake.Sent(), 1)
+ require.Equal(t, "MSG1", fake.Sent()[0].id)
+ require.Empty(t, fake.Sent()[0].text)
+
+ messages := fake.ImageSends()
+ require.Len(t, messages, 1)
+
+ message := messages[0]
+ require.Equal(t, "The plan.", message.GetCaption())
+ require.Equal(t, "image/png", message.GetMimetype())
+ require.Equal(t, "https://example.com/image", message.GetURL())
+ require.Equal(t, "/image", message.GetDirectPath())
+ require.Equal(t, []byte("made-up key"), message.GetMediaKey())
+ require.Equal(t, []byte("made-up hash"), message.GetFileSHA256())
+ require.Equal(t, []byte("made-up encrypted hash"), message.GetFileEncSHA256())
+ require.Equal(t, uint64(len(picture)), message.GetFileLength())
+
+ state, err := openWhatsAppState(whatsapp.state.path)
+ require.NoError(t, err)
+
+ record, found := state.Send("photo")
+ require.True(t, found)
+ require.Equal(t, name, record.Image)
+ require.Equal(t, whatsappSent, record.State)
+
+ again, err := whatsappSend(t.Context(), whatsapp.link(), whatsapp.groups, state, whatsapp.inbox.images, whatsapp.clock, "photo", whatsappTestAna.String(), "The plan.", name)
+ require.NoError(t, err)
+ require.Contains(t, again, "nothing was sent again")
+ require.Len(t, fake.Uploads(), 1)
+ require.Len(t, fake.Sent(), 1)
+
+ other, err := whatsapp.inbox.images.Save(imageTestPNG(t, 1, 1))
+ require.NoError(t, err)
+
+ _, err = whatsapp.Send(t.Context(), "photo", whatsappTestAna.String(), "The plan.", other)
+ require.ErrorContains(t, err, "belongs to another message")
+
+ _, err = whatsapp.Send(t.Context(), "photo", whatsappTestAna.String(), "The plan.", "")
+ require.ErrorContains(t, err, "belongs to another message")
+ require.Len(t, fake.Uploads(), 1)
+}
+
+func TestWhatsAppImageSendAllowsEmptyCaptionAndRefusesUnknownImage(t *testing.T) {
+ whatsapp, fake := whatsappStartTestLinked(t)
+ daemonStartTestWithWhatsApp(t, newModelFake(t), nil, whatsapp)
+ whatsappTestConnect(t, fake)
+
+ _, err := whatsapp.Send(t.Context(), "unknown", whatsappTestAna.String(), "A caption.", "../image.png")
+ require.ErrorContains(t, err, "invalid image name")
+
+ _, found := whatsapp.state.Send("unknown")
+ require.False(t, found)
+ require.Empty(t, fake.Uploads())
+ require.Empty(t, fake.Sent())
+
+ name, err := whatsapp.inbox.images.Save(imageTestPNG(t, 3, 2))
+ require.NoError(t, err)
+
+ _, err = whatsapp.Send(t.Context(), "no-caption", whatsappTestAna.String(), "", name)
+ require.NoError(t, err)
+ require.Len(t, fake.ImageSends(), 1)
+ require.Empty(t, fake.ImageSends()[0].GetCaption())
+}
+
+func TestWhatsAppImageFailureKeepsNoSecretsAndDoesNotRetry(t *testing.T) {
+ testCases := []struct {
+ name string
+ upload bool
+ }{
+ {
+ name: "upload failure stores a safe cause",
+ upload: true,
+ },
+ {name: "send failure stores a safe cause"},
+ }
+ for _, tc := range testCases {
+ t.Run(tc.name, func(t *testing.T) {
+ whatsapp, fake := whatsappStartTestLinked(t)
+ failure := errors.New("https://example.com/media?token=secret /private-media/key phone=" + whatsappTestNumber)
+ if tc.upload {
+ fake.uploadErr = failure
+ } else {
+ fake.sendErr = failure
+ }
+
+ daemonStartTestWithWhatsApp(t, newModelFake(t), nil, whatsapp)
+ whatsappTestConnect(t, fake)
+
+ name, err := whatsapp.inbox.images.Save(imageTestPNG(t, 2, 1))
+ require.NoError(t, err)
+
+ for range 2 {
+ answer, err := whatsapp.Send(t.Context(), "failed-photo", whatsappTestAna.String(), "Private caption.", name)
+ require.NoError(t, err)
+ require.Contains(t, answer, "media send failed")
+ require.NotContains(t, answer, "secret")
+ require.NotContains(t, answer, "/private-media")
+ require.NotContains(t, answer, whatsappTestNumber)
+ require.NotContains(t, answer, "Private caption")
+ }
+
+ require.Len(t, fake.Uploads(), 1)
+
+ if tc.upload {
+ require.Empty(t, fake.Sent())
+ } else {
+ require.Len(t, fake.Sent(), 1)
+ }
+
+ record, found := whatsapp.state.Send("failed-photo")
+ require.True(t, found)
+ require.Equal(t, whatsappFailed, record.State)
+ require.NotContains(t, record.Answer, "secret")
+ })
+ }
+}
+
+func TestWhatsAppImageInProgressRecordDoesNotUploadAgain(t *testing.T) {
+ whatsapp, fake := whatsappStartTestLinked(t)
+ daemonStartTestWithWhatsApp(t, newModelFake(t), nil, whatsapp)
+ whatsappTestConnect(t, fake)
+
+ name, err := whatsapp.inbox.images.Save(imageTestPNG(t, 2, 2))
+ require.NoError(t, err)
+
+ digest := sha256.Sum256([]byte("The plan."))
+ require.NoError(t, whatsapp.state.PutSend(WhatsAppSend{
+ ID: "interrupted-image",
+ Chat: whatsappTestAna.String(),
+ Digest: hex.EncodeToString(digest[:]),
+ Image: name,
+ State: whatsappSending,
+ }))
+
+ answer, err := whatsapp.Send(t.Context(), "interrupted-image", whatsappTestAna.String(), "The plan.", name)
+ require.NoError(t, err)
+ require.Contains(t, answer, "its result is unknown")
+ require.Empty(t, fake.Uploads())
+ require.Empty(t, fake.Sent())
+}
+
+func TestTelegramReceivedPhotoCanBeShownAndSentToWhatsApp(t *testing.T) {
+ whatsapp, fake := whatsappStartTestLinked(t)
+ daemonStartTestWithWhatsApp(t, newModelFake(t), nil, whatsapp)
+ whatsappTestConnect(t, fake)
+
+ telegram := newTelegramFake(t)
+ picture := imageTestPNG(t, 5, 5)
+ telegram.Image("received", picture)
+
+ api := newTelegramAPI(telegram.URL, telegramTestToken)
+ text, err := telegramImageText(t.Context(), api, whatsapp.inbox.images, &TelegramMessage{
+ Caption: "The new plan.",
+ Photo: []TelegramPhoto{{
+ FileID: "received",
+ Width: 5,
+ Height: 5,
+ }},
+ })
+ require.NoError(t, err)
+
+ names := imageFiles(text)
+ require.Len(t, names, 1)
+
+ bot, logs := telegramStartTestBot(t, telegram)
+ box := &Toolbox{
+ whatsapp: whatsapp,
+ imageShow: &ImageShow{
+ store: whatsapp.inbox.images,
+ bot: bot,
+ },
+ }
+ require.NotContains(t, box.run(t.Context(), "image_show", `{"image":"`+names[0]+`","caption":"The plan."}`), "error:")
+ require.Contains(t, box.run(t.Context(), "whatsapp_send", `{"id":"forward","chat":"15550002222@s.whatsapp.net","text":"The plan.","image":"`+names[0]+`"}`), "Sent to")
+ require.Equal(t, [][]byte{picture}, fake.Uploads())
+ require.Equal(t, "The plan.", fake.ImageSends()[0].GetCaption())
+
+ calls := telegram.Calls()
+ require.Equal(t, "sendPhoto", calls[len(calls)-1].method)
+ require.Equal(t, picture, calls[len(calls)-1].params["photo"])
+ require.Empty(t, logs.String())
+}
+
+func TestWhatsAppImageUploadEndsWithItsCallerOrTimeout(t *testing.T) {
+ synctest.Test(t, func(t *testing.T) {
+ client := &WhatsAppUploadWaitFake{WhatsAppFake: &WhatsAppFake{}}
+ start := time.Now()
+ _, err := whatsappImageMessage(t.Context(), client, imageTestPNG(t, 1, 1), "")
+ require.ErrorIs(t, err, context.DeadlineExceeded)
+ require.Equal(t, 2*time.Minute, time.Since(start))
+
+ ctx, cancel := context.WithCancel(t.Context())
+ defer cancel()
+
+ cancel()
+
+ start = time.Now()
+ _, err = whatsappImageMessage(ctx, client, imageTestPNG(t, 1, 1), "")
+ require.ErrorIs(t, err, context.Canceled)
+ require.Zero(t, time.Since(start))
+ })
+}
+
+type WhatsAppUploadWaitFake struct {
+ *WhatsAppFake
+}
+
+func (f *WhatsAppUploadWaitFake) Upload(ctx context.Context, _ []byte, _ whatsmeow.MediaType) (whatsmeow.UploadResponse, error) {
+ <-ctx.Done()
+
+ return whatsmeow.UploadResponse{}, ctx.Err()
+}
}
// WhatsAppSend is the record of one send: the ID that the agent gave, the chat,
-// the SHA-256 of the text, the WhatsApp message ID, the state, which is
-// sending, sent, or failed, the answer of the tool, and the time of its start.
+// the SHA-256 of the text, an optional stored image name, the WhatsApp message
+// ID, the state, which is sending, sent, or failed, the answer of the tool, and
+// the time of its start.
type WhatsAppSend struct {
ID string `json:"id"`
Chat string `json:"chat"`
Digest string `json:"digest"`
+ Image string `json:"image,omitempty"`
MessageID string `json:"message_id"`
State string `json:"state"`
Answer string `json:"answer,omitempty"`
projects / onechat / commitdiff