From: Anderson A. Giacomolli Date: Tue, 10 Nov 2026 19:11:27 +0000 (-0300) Subject: Send stored images to WhatsApp chats X-Git-Url: https://git.giacomolli.uk/onechat/commitdiff_plain/b1be0ee8530df5db8973bfe355f2f6d3149383a0?ds=inline;hp=3cdbd017da9ae7a7045f4272f16d4b262fd01c79 Send stored images to WhatsApp chats WhatsApp sends accept a stored image and use the text as its caption. The send record includes the image name and precedes the upload, so a repeated ID cannot upload or send again. Media errors keep safe causes without URLs or keys. The root asks for consent as for text sends, and image markers in tell give children the shared pixels. The user can also ask to see a received Telegram photo without a separate import. --- diff --git a/README.rst b/README.rst index a02b870..bbb4475 100644 --- a/README.rst +++ b/README.rst @@ -196,7 +196,7 @@ WhatsApp Images Send a photo in Telegram, WhatsApp, or with ``/image`` to the agent. Its memory keeps what the image shows, and it can look at an older - image again. + image again. Ask it to show an image or send one to a WhatsApp chat. Schedules Ask for reminders and timed tasks, such as "every weekday at 8, tell diff --git a/image_show_test.go b/image_show_test.go index ed475fe..65e422c 100644 --- a/image_show_test.go +++ b/image_show_test.go @@ -1,6 +1,7 @@ package main import ( + "encoding/base64" "encoding/json" "net/http" "path/filepath" @@ -134,3 +135,97 @@ func TestAgentShowsImageInTerminalWithoutTelegram(t *testing.T) { Text: output, }), filepath.Join(test.agent.Images().dir, name)) } + +func TestTellImageMarkerGivesTheChildPixels(t *testing.T) { + server := newModelFake(t) + test := newDaemonMessageTest(t, t.TempDir(), server) + picture := imageTestPNG(t, 4, 2) + name, err := test.agents[0].agent.Images().Save(picture) + require.NoError(t, err) + test.Start(t, 0, 1) + + arguments, err := json.Marshal(ToolTellArguments{ + Name: "Garden", + Message: imageText([]string{name}, "Review this plan."), + }) + require.NoError(t, err) + require.Contains(t, test.loops[0].tools.run(t.Context(), "tell", string(arguments)), "Sent.") + test.events[1].WaitFor(t, "idle", 1) + + var childCall *ModelFakeCall + + for i := range server.Calls() { + call := server.Call(i) + if call.body["model"] == "gpt-test-garden" { + childCall = &call + break + } + } + + require.NotNil(t, childCall) + require.Contains(t, string(childCall.raw), "input_image") + require.Contains(t, string(childCall.raw), base64.StdEncoding.EncodeToString(picture)) + require.Contains(t, test.events[1].Last("user"), "[image "+name+"]") + require.NotContains(t, promptToolNames(test.agents[1].agent.Tools()), "image_show") +} + +func TestAgentShowsTelegramReceivedImageToItsOwner(t *testing.T) { + picture := imageTestPNG(t, 6, 4) + store := &ImageStore{dir: t.TempDir()} + name, err := store.Save(picture) + require.NoError(t, err) + + telegram := newTelegramFake(t) + telegram.Image("owner-photo", picture) + + bot, logs := telegramStartTestBot(t, telegram) + server := newModelFake(t) + server.reply = func(w http.ResponseWriter, call ModelFakeCall) { + if call.number == 0 { + arguments, err := json.Marshal(ToolImageShowArguments{ + Image: name, + Caption: "Here is your photo.", + }) + require.NoError(t, err) + modelWriteEvents(w, modelCompletedEvent([]any{modelFunctionCallItem("fc_1", "call_1", "image_show", string(arguments))}, 100, 0, 5)) + + return + } + + modelWriteEvents(w, modelCompletedEvent([]any{modelMessageItem("Shown.")}, 100, 0, 5)) + } + socket := daemonStartTestWithBot(t, server, bot) + terminal := terminalStartTestClient(t, socket) + terminal.WaitFor(t, "[connected to OneChat]") + + message := telegramOwnerMessage("") + message.Caption = "Show this photo." + message.Photo = []TelegramPhoto{{ + FileID: "owner-photo", + Width: 6, + Height: 4, + }} + telegram.Add(TelegramUpdate{ + UpdateID: 1, + Message: message, + }) + terminal.WaitFor(t, "Shown.\n[done]") + require.Contains(t, terminal.Output(), "[result] [image "+name+"]") + require.Contains(t, terminal.Output(), filepath.Join(imageDirName, name)) + + var photos []TelegramFakeCall + + for _, call := range telegram.Calls() { + if call.method == "sendPhoto" { + photos = append(photos, call) + } + } + + require.Len(t, photos, 1) + require.Equal(t, picture, photos[0].params["photo"]) + require.Equal(t, "4242", photos[0].params["chat_id"]) + require.Equal(t, "Here is your photo.", photos[0].params["caption"]) + require.NotContains(t, logs.String(), "Here is your photo.") + require.NotContains(t, logs.String(), "Show this photo.") + require.NotContains(t, logs.String(), telegramTestToken) +} diff --git a/prompt.go b/prompt.go index 5847ee8..0852bec 100644 --- a/prompt.go +++ b/prompt.go @@ -154,7 +154,9 @@ Pass area messages from mixed chats to the right child with sender, date, and words. Never send a WhatsApp reply drafted by a child without the user's agreement. Propose routing a source that talks only about one area to its child. -Set or remove a route only after the user agrees.` +Set or remove a route only after the user agrees. +Use image_show or whatsapp_send with an image only on the user's request. +Pass an image to a child with an [image FILE] line in tell.` // promptSubagentText is the system prompt of a subagent, from the subagent // prompt of the reference design: its head, then the doc of the view, without @@ -294,11 +296,12 @@ func promptTools() []ModelTool { }, ModelTool{ Type: "function", Name: "whatsapp_send", - Description: "Send a text message to a WhatsApp chat. Send only a message that the user asked you to send, to the chat that the user named. Give a new id for each message: a call with an id that was used gives the answer of the first call and sends nothing.", + Description: "Send text or a stored image with text as its caption to a WhatsApp chat. Send only a message that the user asked you to send, to the chat that the user named. Give a new id for each message: a call with an id that was used gives the answer of the first call and sends nothing.", Parameters: promptObjectSchema(map[string]any{ - "id": promptStringSchema("A new id for this message, such as a short random word."), - "chat": promptStringSchema("The ID of the chat, from whatsapp_find or from a [WhatsApp] message."), - "text": promptStringSchema("The text of the message."), + "id": promptStringSchema("A new id for this message, such as a short random word."), + "chat": promptStringSchema("The ID of the chat, from whatsapp_find or from a [WhatsApp] message."), + "text": promptStringSchema("The text, or the caption when sending an image. Can be empty with an image."), + "image": promptNullableStringSchema("The stored image filename from an [image FILE] line, or null for text only."), }), Strict: true, }, ModelTool{ diff --git a/prompt_test.go b/prompt_test.go index 06fea49..888057f 100644 --- a/prompt_test.go +++ b/prompt_test.go @@ -166,3 +166,14 @@ func TestRootPromptDefinesChildRoutingAndConsent(t *testing.T) { require.NotContains(t, child, "Use agent_create") require.Contains(t, child, "Send nothing to WhatsApp. Draft replies in your report instead") } + +func TestRootPromptKeepsImageConsentAndChildForwardingRules(t *testing.T) { + text := prompt("Ada", "/AGENT/instructions.md", "") + require.Contains(t, text, "Use image_show or whatsapp_send with an image only on the user's request.") + require.Contains(t, text, "Pass an image to a child with an [image FILE] line in tell.") + + child := promptChild("House", "Ada", "Keep house plans.", "/CHILD/instructions.md", "") + require.Contains(t, child, "Send nothing to WhatsApp. Draft replies in your report instead") + require.NotContains(t, promptToolNames(promptSubagentTools()), "image_show") + require.NotContains(t, promptToolNames(promptSubagentTools()), "whatsapp_send") +} diff --git a/tools.go b/tools.go index 9f2905a..621cd84 100644 --- a/tools.go +++ b/tools.go @@ -260,7 +260,7 @@ func (b *Toolbox) run(ctx context.Context, name, arguments string) string { case b.whatsapp == nil: err = fmt.Errorf("unknown tool %q", name) case err == nil: - output, err = b.whatsapp.Send(ctx, args.ID, args.Chat, args.Text) + output, err = b.whatsapp.Send(ctx, args.ID, args.Chat, args.Text, args.Image) } case "schedule_set": var args ToolScheduleSetArguments @@ -573,9 +573,10 @@ type ToolWhatsAppFindArguments struct { // ToolWhatsAppSendArguments are the arguments of whatsapp_send. type ToolWhatsAppSendArguments struct { - ID string `json:"id"` - Chat string `json:"chat"` - Text string `json:"text"` + ID string `json:"id"` + Chat string `json:"chat"` + Text string `json:"text"` + Image string `json:"image"` } // ToolScheduleSetArguments are the arguments of schedule_set. Exactly one of At diff --git a/whatsapp.go b/whatsapp.go index 97b1e85..57bfab0 100644 --- a/whatsapp.go +++ b/whatsapp.go @@ -92,6 +92,7 @@ type WhatsAppClient interface { Logout(ctx context.Context) error GetJoinedGroups(ctx context.Context) ([]*types.GroupInfo, error) Download(ctx context.Context, message whatsmeow.DownloadableMessage) ([]byte, error) + Upload(ctx context.Context, data []byte, mediaType whatsmeow.MediaType) (whatsmeow.UploadResponse, error) GenerateMessageID() types.MessageID SendMessage(ctx context.Context, to types.JID, message *waE2E.Message, extra ...whatsmeow.SendRequestExtra) (whatsmeow.SendResponse, error) } @@ -462,10 +463,10 @@ func (w *WhatsApp) Find(ctx context.Context, name string) (string, error) { return whatsappFind(ctx, w.link(), w.groups, name) } -// Send returns the answer of whatsapp_send for the send id of text to chat, as -// whatsappSend says. -func (w *WhatsApp) Send(ctx context.Context, id, chat, text string) (string, error) { - return whatsappSend(ctx, w.link(), w.groups, w.state, w.clock, id, chat, text) +// Send returns the answer of whatsapp_send for text and an optional stored +// image to chat, with the durable send id that whatsappSend describes. +func (w *WhatsApp) Send(ctx context.Context, id, chat, text, image string) (string, error) { + return whatsappSend(ctx, w.link(), w.groups, w.state, w.inbox.images, w.clock, id, chat, text, image) } func (w *WhatsApp) sessionPath() string { diff --git a/whatsapp_fake_helper_test.go b/whatsapp_fake_helper_test.go index 9f80b38..8e0f237 100644 --- a/whatsapp_fake_helper_test.go +++ b/whatsapp_fake_helper_test.go @@ -84,6 +84,9 @@ type WhatsAppFake struct { logoutErr error groups []*types.GroupInfo sendErr error + uploadErr error + uploads [][]byte + imageSends []*waE2E.ImageMessage downloadData []byte downloadErr error downloadWait time.Duration @@ -203,6 +206,46 @@ func (f *WhatsAppFake) Downloads() []whatsmeow.DownloadableMessage { return slices.Clone(f.downloads) } +// Upload records the pixels and returns made-up media metadata. +func (f *WhatsAppFake) Upload(_ context.Context, data []byte, mediaType whatsmeow.MediaType) (whatsmeow.UploadResponse, error) { + f.mutex.Lock() + defer f.mutex.Unlock() + + if mediaType != whatsmeow.MediaImage { + return whatsmeow.UploadResponse{}, fmt.Errorf("unexpected media type %q", mediaType) + } + + f.uploads = append(f.uploads, slices.Clone(data)) + if f.uploadErr != nil { + return whatsmeow.UploadResponse{}, f.uploadErr + } + + return whatsmeow.UploadResponse{ + URL: "https://example.com/image", + DirectPath: "/image", + MediaKey: []byte("made-up key"), + FileSHA256: []byte("made-up hash"), + FileEncSHA256: []byte("made-up encrypted hash"), + FileLength: uint64(len(data)), + }, nil +} + +// Uploads returns the pixels passed to Upload. +func (f *WhatsAppFake) Uploads() [][]byte { + f.mutex.Lock() + defer f.mutex.Unlock() + + return slices.Clone(f.uploads) +} + +// ImageSends returns the outgoing image messages. +func (f *WhatsAppFake) ImageSends() []*waE2E.ImageMessage { + f.mutex.Lock() + defer f.mutex.Unlock() + + return slices.Clone(f.imageSends) +} + // GenerateMessageID returns a new message ID. func (f *WhatsAppFake) GenerateMessageID() types.MessageID { f.mutex.Lock() @@ -216,6 +259,10 @@ func (f *WhatsAppFake) SendMessage(_ context.Context, to types.JID, message *waE f.mutex.Lock() defer f.mutex.Unlock() + if message.GetImageMessage() != nil { + f.imageSends = append(f.imageSends, message.GetImageMessage()) + } + f.sent = append(f.sent, WhatsAppFakeSend{ to: to.String(), text: message.GetConversation(), diff --git a/whatsapp_image.go b/whatsapp_image.go index d4b88d8..5ce13a8 100644 --- a/whatsapp_image.go +++ b/whatsapp_image.go @@ -4,11 +4,16 @@ import ( "context" "errors" "fmt" + "net/http" + "time" "go.mau.fi/whatsmeow" "go.mau.fi/whatsmeow/proto/waE2E" ) +// whatsappImageUploadTimeout limits one image upload to WhatsApp. +const whatsappImageUploadTimeout = 2 * time.Minute + // whatsappImage downloads only supported images and stores them before the // event can be acknowledged. Both the declared size and the downloaded bytes // must fit the image store. Decoded bytes determine the stored format. @@ -59,3 +64,39 @@ func whatsappImageDownloadError(err error) error { return errors.New("media download failed") } + +// whatsappImageMessage uploads stored pixels and uses the upload keys only in +// the outgoing message. The text is the caption, not a separate message. +func whatsappImageMessage(ctx context.Context, client WhatsAppClient, data []byte, text string) (*waE2E.Message, error) { + ctx, cancel := context.WithTimeout(ctx, whatsappImageUploadTimeout) + defer cancel() + + upload, err := client.Upload(ctx, data, whatsmeow.MediaImage) + if err != nil { + return nil, fmt.Errorf("upload image: %w", whatsappImageSendError(err)) + } + + return &waE2E.Message{ImageMessage: &waE2E.ImageMessage{ + Caption: new(text), + Mimetype: new(http.DetectContentType(data)), + URL: &upload.URL, + DirectPath: &upload.DirectPath, + MediaKey: upload.MediaKey, + FileEncSHA256: upload.FileEncSHA256, + FileSHA256: upload.FileSHA256, + FileLength: &upload.FileLength, + }}, nil +} + +// whatsappImageSendError drops arbitrary upload and send errors, which can hold +// media URLs and keys. Cancellation and deadline causes are safe to keep. +func whatsappImageSendError(err error) error { + switch { + case errors.Is(err, context.Canceled): + return context.Canceled + case errors.Is(err, context.DeadlineExceeded): + return context.DeadlineExceeded + default: + return errors.New("media send failed") + } +} diff --git a/whatsapp_send.go b/whatsapp_send.go index d5995df..b95a82e 100644 --- a/whatsapp_send.go +++ b/whatsapp_send.go @@ -31,21 +31,24 @@ const ( whatsappFailed = "failed" ) -// whatsappSend sends text to chat, the ID of a private chat or a group, and -// returns the answer of the tool. id names the send: a call with an id that has -// a record gives the answer of the record and sends nothing, and the same id -// with another chat or text is an error. The record "sending" is durable before -// the send, so a stop of the daemon during the send gives an answer that the -// result is unknown, and not a second send. A disconnected WhatsApp sends -// nothing and makes no record. link gives the device, the client, and the own -// IDs, groups the names of the groups, and state the send records. The clock -// gives the times of the records and the time zone of the answer. -func whatsappSend(ctx context.Context, link WhatsAppLink, groups *WhatsAppGroups, state *WhatsAppState, clock func() time.Time, id, chat, text string) (string, error) { +// whatsappSend sends text and an optional stored image to chat, the ID of a +// private chat or a group, and returns the answer of the tool. id names the +// send: a call with an id that has a record gives the answer of the record and +// sends nothing, and the same id with another chat, text, or image is an error. +// The record "sending" is durable before the send, so a stop of the daemon +// during the send gives an answer that the result is unknown, and not a second +// send. A disconnected WhatsApp sends nothing and makes no record. link gives +// the device, the client, and the own IDs, groups the names of the groups, and +// state the send records. The clock gives the times of the records and the time +// zone of the answer. Image names are checked before the record is written, and +// uploads start only after the record is durable. Text-only records keep their +// text digest. +func whatsappSend(ctx context.Context, link WhatsAppLink, groups *WhatsAppGroups, state *WhatsAppState, images *ImageStore, clock func() time.Time, id, chat, text, image string) (string, error) { if !link.linked { return whatsappNotLinked, nil } - jid, err := whatsappSendCheck(id, chat, text, link.own) + jid, err := whatsappSendCheck(id, chat, text, image, link.own) if err != nil { return "", err } @@ -53,6 +56,7 @@ func whatsappSend(ctx context.Context, link WhatsAppLink, groups *WhatsAppGroups digest := sha256.Sum256([]byte(text)) record := WhatsAppSend{ ID: id, + Image: image, Chat: chat, Digest: hex.EncodeToString(digest[:]), State: whatsappSending, @@ -61,7 +65,7 @@ func whatsappSend(ctx context.Context, link WhatsAppLink, groups *WhatsAppGroups held, found := state.Send(id) switch { - case found && (held.Chat != record.Chat || held.Digest != record.Digest): + case found && (held.Chat != record.Chat || held.Digest != record.Digest || held.Image != record.Image): return "", fmt.Errorf("the id %s belongs to another message: give a new id for each message", id) case found && held.State == whatsappSending: return "The send with this id started, but its result is unknown, because the daemon stopped during it. Ask the user to look at the chat in WhatsApp before you send the message again with a new id.", nil @@ -71,20 +75,39 @@ func whatsappSend(ctx context.Context, link WhatsAppLink, groups *WhatsAppGroups return "", errors.New("WhatsApp is disconnected, so nothing was sent: try again later") } + var data []byte + + if image != "" { + data, err = images.Read(image) + if err != nil { + return "", fmt.Errorf("read image to send: %w", err) + } + } + record.MessageID = link.client.GenerateMessageID() err = state.PutSend(record) if err != nil { return "", err } + message := &waE2E.Message{Conversation: new(text)} + if image != "" { + message, err = whatsappImageMessage(ctx, link.client, data, text) + } + sendCtx, cancel := context.WithTimeout(ctx, whatsappSendTimeout) defer cancel() - response, err := link.client.SendMessage(sendCtx, jid, &waE2E.Message{ - Conversation: new(text), - }, whatsmeow.SendRequestExtra{ - ID: record.MessageID, - }) + var response whatsmeow.SendResponse + + if err == nil { + response, err = link.client.SendMessage(sendCtx, jid, message, whatsmeow.SendRequestExtra{ + ID: record.MessageID, + }) + if err != nil && image != "" { + err = fmt.Errorf("send image: %w", whatsappImageSendError(err)) + } + } record.State = whatsappSent if err != nil { @@ -108,14 +131,14 @@ func whatsappSend(ctx context.Context, link WhatsAppLink, groups *WhatsAppGroups } // whatsappSendCheck returns the JID of chat, and an error when id, chat, or -// text cannot go to a send. The chat is a private chat or a group, and not the -// chat of the user with itself. -func whatsappSendCheck(id, chat, text string, own []string) (types.JID, error) { +// text cannot go to a send. Text can be empty only with an image. The chat is a +// private chat or a group, and not the chat of the user with itself. +func whatsappSendCheck(id, chat, text, image string, own []string) (types.JID, error) { jid, err := types.ParseJID(chat) switch { case strings.TrimSpace(id) == "" || len(id) > whatsappSendIDBytes: return types.EmptyJID, fmt.Errorf("the id must have 1 to %d bytes", whatsappSendIDBytes) - case strings.TrimSpace(text) == "" || len(text) > whatsappSendBytes: + case (strings.TrimSpace(text) == "" && image == "") || len(text) > whatsappSendBytes: return types.EmptyJID, fmt.Errorf("the text must have 1 to %d bytes", whatsappSendBytes) case err != nil || jid.User == "" || jid != jid.ToNonAD() || jid.String() != chat: return types.EmptyJID, fmt.Errorf("%q is not the ID of a WhatsApp chat: find the ID with whatsapp_find", chat) diff --git a/whatsapp_send_test.go b/whatsapp_send_test.go index 480e7db..65c2fc6 100644 --- a/whatsapp_send_test.go +++ b/whatsapp_send_test.go @@ -1,6 +1,7 @@ package main import ( + "context" "crypto/sha256" "encoding/hex" "errors" @@ -8,6 +9,10 @@ import ( "path/filepath" "strings" "testing" + "testing/synctest" + "time" + + "go.mau.fi/whatsmeow" "github.com/stretchr/testify/require" "go.mau.fi/whatsmeow/types" @@ -20,15 +25,15 @@ func TestWhatsAppSendSendsOnceForEachID(t *testing.T) { daemonStartTestWithWhatsApp(t, newModelFake(t), nil, whatsapp) whatsappTestConnect(t, fake) - answer, err := whatsapp.Send(t.Context(), "keys", whatsappTestAna.String(), "The keys are under the mat.") + answer, err := whatsapp.Send(t.Context(), "keys", whatsappTestAna.String(), "The keys are under the mat.", "") require.NoError(t, err) require.Equal(t, "Sent to Ana Souza at 12:05.", answer) - answer, err = whatsapp.Send(t.Context(), "keys", whatsappTestAna.String(), "The keys are under the mat.") + answer, err = whatsapp.Send(t.Context(), "keys", whatsappTestAna.String(), "The keys are under the mat.", "") require.NoError(t, err) require.Equal(t, "Sent to Ana Souza at 12:05. This is the answer of the first call with this id: nothing was sent again.", answer) - _, err = whatsapp.Send(t.Context(), "keys", whatsappTestAna.String(), "Another text.") + _, err = whatsapp.Send(t.Context(), "keys", whatsappTestAna.String(), "Another text.", "") require.EqualError(t, err, "the id keys belongs to another message: give a new id for each message") require.Equal(t, []WhatsAppFakeSend{ @@ -62,7 +67,7 @@ func TestWhatsAppSendToAGroup(t *testing.T) { daemonStartTestWithWhatsApp(t, newModelFake(t), nil, whatsapp) whatsappTestConnect(t, fake) - answer, err := whatsapp.Send(t.Context(), "cake", whatsappTestFamily.String(), "I bring the cake.") + answer, err := whatsapp.Send(t.Context(), "cake", whatsappTestFamily.String(), "I bring the cake.", "") require.NoError(t, err) require.Equal(t, "Sent to Family at 12:05.", answer) } @@ -75,7 +80,7 @@ func TestWhatsAppSendThatFails(t *testing.T) { whatsappTestConnect(t, fake) for range 2 { - answer, err := whatsapp.Send(t.Context(), "keys", whatsappTestAna.String(), "hi") + answer, err := whatsapp.Send(t.Context(), "keys", whatsappTestAna.String(), "hi", "") require.NoError(t, err) require.True(t, strings.HasPrefix(answer, "The send failed: server error for OWN."), answer) } @@ -96,7 +101,7 @@ func TestWhatsAppSendCutByAStop(t *testing.T) { State: whatsappSending, })) - answer, err := whatsapp.Send(t.Context(), "keys", whatsappTestAna.String(), "hi") + answer, err := whatsapp.Send(t.Context(), "keys", whatsappTestAna.String(), "hi", "") require.NoError(t, err) require.Contains(t, answer, "its result is unknown") require.Empty(t, fake.Sent()) @@ -153,7 +158,7 @@ func TestWhatsAppSendRefuses(t *testing.T) { for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { - _, err := whatsapp.Send(t.Context(), tc.id, tc.chat, tc.text) + _, err := whatsapp.Send(t.Context(), tc.id, tc.chat, tc.text, "") require.EqualError(t, err, tc.err) }) } @@ -167,12 +172,12 @@ func TestWhatsAppSendWhileDisconnected(t *testing.T) { whatsappTestConnect(t, fake) require.True(t, fake.Emit(&events.Disconnected{})) - _, err := whatsapp.Send(t.Context(), "keys", whatsappTestAna.String(), "hi") + _, err := whatsapp.Send(t.Context(), "keys", whatsappTestAna.String(), "hi", "") require.EqualError(t, err, "WhatsApp is disconnected, so nothing was sent: try again later") require.True(t, fake.Emit(&events.Connected{})) - answer, err := whatsapp.Send(t.Context(), "keys", whatsappTestAna.String(), "hi") + answer, err := whatsapp.Send(t.Context(), "keys", whatsappTestAna.String(), "hi", "") require.NoError(t, err) require.Equal(t, "Sent to +15550002222 at 12:05.", answer) } @@ -180,7 +185,7 @@ func TestWhatsAppSendWhileDisconnected(t *testing.T) { func TestWhatsAppSendWithoutALink(t *testing.T) { whatsapp, _, _ := whatsappStartTest(t, t.TempDir(), newTestClock()) - answer, err := whatsapp.Send(t.Context(), "keys", whatsappTestAna.String(), "hi") + answer, err := whatsapp.Send(t.Context(), "keys", whatsappTestAna.String(), "hi", "") require.NoError(t, err) require.Equal(t, whatsappNotLinked, answer) } @@ -208,3 +213,231 @@ func TestAgentSendsAWhatsAppMessage(t *testing.T) { client.WaitFor(t, "[result] Sent to Ana Souza at 12:05.\nSent.\n[done]") require.Len(t, fake.Sent(), 1) } + +func TestWhatsAppImageSendUploadsOnceAndKeepsCaption(t *testing.T) { + whatsapp, fake := whatsappStartTestLinked(t) + daemonStartTestWithWhatsApp(t, newModelFake(t), nil, whatsapp) + whatsappTestConnect(t, fake) + + picture := imageTestPNG(t, 4, 3) + name, err := whatsapp.inbox.images.Save(picture) + require.NoError(t, err) + + box := &Toolbox{whatsapp: whatsapp} + arguments := `{"id":"photo","chat":"15550002222@s.whatsapp.net","text":"The plan.","image":"` + name + `"}` + answer := box.run(t.Context(), "whatsapp_send", arguments) + require.Contains(t, answer, "Sent to") + require.Contains(t, box.run(t.Context(), "whatsapp_send", arguments), "nothing was sent again") + require.Equal(t, [][]byte{picture}, fake.Uploads()) + require.Len(t, fake.Sent(), 1) + require.Equal(t, "MSG1", fake.Sent()[0].id) + require.Empty(t, fake.Sent()[0].text) + + messages := fake.ImageSends() + require.Len(t, messages, 1) + + message := messages[0] + require.Equal(t, "The plan.", message.GetCaption()) + require.Equal(t, "image/png", message.GetMimetype()) + require.Equal(t, "https://example.com/image", message.GetURL()) + require.Equal(t, "/image", message.GetDirectPath()) + require.Equal(t, []byte("made-up key"), message.GetMediaKey()) + require.Equal(t, []byte("made-up hash"), message.GetFileSHA256()) + require.Equal(t, []byte("made-up encrypted hash"), message.GetFileEncSHA256()) + require.Equal(t, uint64(len(picture)), message.GetFileLength()) + + state, err := openWhatsAppState(whatsapp.state.path) + require.NoError(t, err) + + record, found := state.Send("photo") + require.True(t, found) + require.Equal(t, name, record.Image) + require.Equal(t, whatsappSent, record.State) + + again, err := whatsappSend(t.Context(), whatsapp.link(), whatsapp.groups, state, whatsapp.inbox.images, whatsapp.clock, "photo", whatsappTestAna.String(), "The plan.", name) + require.NoError(t, err) + require.Contains(t, again, "nothing was sent again") + require.Len(t, fake.Uploads(), 1) + require.Len(t, fake.Sent(), 1) + + other, err := whatsapp.inbox.images.Save(imageTestPNG(t, 1, 1)) + require.NoError(t, err) + + _, err = whatsapp.Send(t.Context(), "photo", whatsappTestAna.String(), "The plan.", other) + require.ErrorContains(t, err, "belongs to another message") + + _, err = whatsapp.Send(t.Context(), "photo", whatsappTestAna.String(), "The plan.", "") + require.ErrorContains(t, err, "belongs to another message") + require.Len(t, fake.Uploads(), 1) +} + +func TestWhatsAppImageSendAllowsEmptyCaptionAndRefusesUnknownImage(t *testing.T) { + whatsapp, fake := whatsappStartTestLinked(t) + daemonStartTestWithWhatsApp(t, newModelFake(t), nil, whatsapp) + whatsappTestConnect(t, fake) + + _, err := whatsapp.Send(t.Context(), "unknown", whatsappTestAna.String(), "A caption.", "../image.png") + require.ErrorContains(t, err, "invalid image name") + + _, found := whatsapp.state.Send("unknown") + require.False(t, found) + require.Empty(t, fake.Uploads()) + require.Empty(t, fake.Sent()) + + name, err := whatsapp.inbox.images.Save(imageTestPNG(t, 3, 2)) + require.NoError(t, err) + + _, err = whatsapp.Send(t.Context(), "no-caption", whatsappTestAna.String(), "", name) + require.NoError(t, err) + require.Len(t, fake.ImageSends(), 1) + require.Empty(t, fake.ImageSends()[0].GetCaption()) +} + +func TestWhatsAppImageFailureKeepsNoSecretsAndDoesNotRetry(t *testing.T) { + testCases := []struct { + name string + upload bool + }{ + { + name: "upload failure stores a safe cause", + upload: true, + }, + {name: "send failure stores a safe cause"}, + } + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + whatsapp, fake := whatsappStartTestLinked(t) + failure := errors.New("https://example.com/media?token=secret /private-media/key phone=" + whatsappTestNumber) + if tc.upload { + fake.uploadErr = failure + } else { + fake.sendErr = failure + } + + daemonStartTestWithWhatsApp(t, newModelFake(t), nil, whatsapp) + whatsappTestConnect(t, fake) + + name, err := whatsapp.inbox.images.Save(imageTestPNG(t, 2, 1)) + require.NoError(t, err) + + for range 2 { + answer, err := whatsapp.Send(t.Context(), "failed-photo", whatsappTestAna.String(), "Private caption.", name) + require.NoError(t, err) + require.Contains(t, answer, "media send failed") + require.NotContains(t, answer, "secret") + require.NotContains(t, answer, "/private-media") + require.NotContains(t, answer, whatsappTestNumber) + require.NotContains(t, answer, "Private caption") + } + + require.Len(t, fake.Uploads(), 1) + + if tc.upload { + require.Empty(t, fake.Sent()) + } else { + require.Len(t, fake.Sent(), 1) + } + + record, found := whatsapp.state.Send("failed-photo") + require.True(t, found) + require.Equal(t, whatsappFailed, record.State) + require.NotContains(t, record.Answer, "secret") + }) + } +} + +func TestWhatsAppImageInProgressRecordDoesNotUploadAgain(t *testing.T) { + whatsapp, fake := whatsappStartTestLinked(t) + daemonStartTestWithWhatsApp(t, newModelFake(t), nil, whatsapp) + whatsappTestConnect(t, fake) + + name, err := whatsapp.inbox.images.Save(imageTestPNG(t, 2, 2)) + require.NoError(t, err) + + digest := sha256.Sum256([]byte("The plan.")) + require.NoError(t, whatsapp.state.PutSend(WhatsAppSend{ + ID: "interrupted-image", + Chat: whatsappTestAna.String(), + Digest: hex.EncodeToString(digest[:]), + Image: name, + State: whatsappSending, + })) + + answer, err := whatsapp.Send(t.Context(), "interrupted-image", whatsappTestAna.String(), "The plan.", name) + require.NoError(t, err) + require.Contains(t, answer, "its result is unknown") + require.Empty(t, fake.Uploads()) + require.Empty(t, fake.Sent()) +} + +func TestTelegramReceivedPhotoCanBeShownAndSentToWhatsApp(t *testing.T) { + whatsapp, fake := whatsappStartTestLinked(t) + daemonStartTestWithWhatsApp(t, newModelFake(t), nil, whatsapp) + whatsappTestConnect(t, fake) + + telegram := newTelegramFake(t) + picture := imageTestPNG(t, 5, 5) + telegram.Image("received", picture) + + api := newTelegramAPI(telegram.URL, telegramTestToken) + text, err := telegramImageText(t.Context(), api, whatsapp.inbox.images, &TelegramMessage{ + Caption: "The new plan.", + Photo: []TelegramPhoto{{ + FileID: "received", + Width: 5, + Height: 5, + }}, + }) + require.NoError(t, err) + + names := imageFiles(text) + require.Len(t, names, 1) + + bot, logs := telegramStartTestBot(t, telegram) + box := &Toolbox{ + whatsapp: whatsapp, + imageShow: &ImageShow{ + store: whatsapp.inbox.images, + bot: bot, + }, + } + require.NotContains(t, box.run(t.Context(), "image_show", `{"image":"`+names[0]+`","caption":"The plan."}`), "error:") + require.Contains(t, box.run(t.Context(), "whatsapp_send", `{"id":"forward","chat":"15550002222@s.whatsapp.net","text":"The plan.","image":"`+names[0]+`"}`), "Sent to") + require.Equal(t, [][]byte{picture}, fake.Uploads()) + require.Equal(t, "The plan.", fake.ImageSends()[0].GetCaption()) + + calls := telegram.Calls() + require.Equal(t, "sendPhoto", calls[len(calls)-1].method) + require.Equal(t, picture, calls[len(calls)-1].params["photo"]) + require.Empty(t, logs.String()) +} + +func TestWhatsAppImageUploadEndsWithItsCallerOrTimeout(t *testing.T) { + synctest.Test(t, func(t *testing.T) { + client := &WhatsAppUploadWaitFake{WhatsAppFake: &WhatsAppFake{}} + start := time.Now() + _, err := whatsappImageMessage(t.Context(), client, imageTestPNG(t, 1, 1), "") + require.ErrorIs(t, err, context.DeadlineExceeded) + require.Equal(t, 2*time.Minute, time.Since(start)) + + ctx, cancel := context.WithCancel(t.Context()) + defer cancel() + + cancel() + + start = time.Now() + _, err = whatsappImageMessage(ctx, client, imageTestPNG(t, 1, 1), "") + require.ErrorIs(t, err, context.Canceled) + require.Zero(t, time.Since(start)) + }) +} + +type WhatsAppUploadWaitFake struct { + *WhatsAppFake +} + +func (f *WhatsAppUploadWaitFake) Upload(ctx context.Context, _ []byte, _ whatsmeow.MediaType) (whatsmeow.UploadResponse, error) { + <-ctx.Done() + + return whatsmeow.UploadResponse{}, ctx.Err() +} diff --git a/whatsapp_state.go b/whatsapp_state.go index b1b5935..3d21b16 100644 --- a/whatsapp_state.go +++ b/whatsapp_state.go @@ -298,12 +298,14 @@ type WhatsAppStateFile struct { } // WhatsAppSend is the record of one send: the ID that the agent gave, the chat, -// the SHA-256 of the text, the WhatsApp message ID, the state, which is -// sending, sent, or failed, the answer of the tool, and the time of its start. +// the SHA-256 of the text, an optional stored image name, the WhatsApp message +// ID, the state, which is sending, sent, or failed, the answer of the tool, and +// the time of its start. type WhatsAppSend struct { ID string `json:"id"` Chat string `json:"chat"` Digest string `json:"digest"` + Image string `json:"image,omitempty"` MessageID string `json:"message_id"` State string `json:"state"` Answer string `json:"answer,omitempty"`