From: Anderson A. Giacomolli Date: Tue, 10 Nov 2026 17:14:06 +0000 (-0300) Subject: Receive images in WhatsApp batches X-Git-Url: https://git.giacomolli.uk/onechat/commitdiff_plain/0da50dc187c8b4aa23ac1ddd0418c0da779389d9?ds=inline Receive images in WhatsApp batches WhatsApp stores supported images before it acknowledges their messages, so a stop cannot lose them. A download has its own limit of two minutes, apart from the ten seconds of the name lookups. Batch text keeps the image markers and captions on their own lines, so the agent and a routed child can read the pixels. Failed images and other visual media keep short text markers instead of stopping the batch. View-once messages stay unread. A failed download logs only a short chat hash and a safe cause, such as an HTTP status, because the library error text can hold signed media URLs. Tests use made-up images and a fake download client to check storage, the limits, the markers, routed delivery, and restarts. Issue: 441b3c7 --- diff --git a/README.rst b/README.rst index 1c344f4..a02b870 100644 --- a/README.rst +++ b/README.rst @@ -194,7 +194,7 @@ WhatsApp child of that area, and sets the route after you agree. Images - Send a photo in Telegram or with ``/image``, and the agent sees it. + Send a photo in Telegram, WhatsApp, or with ``/image`` to the agent. Its memory keeps what the image shows, and it can look at an older image again. diff --git a/whatsapp.go b/whatsapp.go index 851cfd1..f225756 100644 --- a/whatsapp.go +++ b/whatsapp.go @@ -95,6 +95,7 @@ type WhatsAppClient interface { PairPhone(ctx context.Context, phone string, showPushNotification bool, clientType whatsmeow.PairClientType, clientDisplayName string) (string, error) Logout(ctx context.Context) error GetJoinedGroups(ctx context.Context) ([]*types.GroupInfo, error) + Download(ctx context.Context, message whatsmeow.DownloadableMessage) ([]byte, error) GenerateMessageID() types.MessageID SendMessage(ctx context.Context, to types.JID, message *waE2E.Message, extra ...whatsmeow.SendRequestExtra) (whatsmeow.SendResponse, error) } @@ -123,7 +124,7 @@ func openWhatsApp(ctx context.Context, dir string, clock func() time.Time, logge return nil, err } - w.inbox = newWhatsAppInbox(w.state, clock, w.groups) + w.inbox = newWhatsAppInbox(w.state, clock, w.groups, &ImageStore{dir: filepath.Join(dir, imageDirName)}, logger) info, err := os.Stat(w.sessionPath()) if errors.Is(err, os.ErrNotExist) { @@ -168,9 +169,9 @@ func openWhatsApp(ctx context.Context, dir string, clock func() time.Time, logge // creation, close, removal, and snapshots. It is held before mutex, and stays // held when removal releases mutex for client callbacks. // -// The inbox puts the text messages into batches and logs them, and groups -// holds the names of the groups. Both get a WhatsAppLink from link for each -// call, so they use the current device and client. +// The inbox puts messages into batches and logs them, and groups holds the +// names of the groups. Both get a WhatsAppLink from link for each call, so they +// use the current device and client. // // The library logs nothing, because its log lines hold the ID of the own // account, which is the phone number of the user. The own number and the own diff --git a/whatsapp_batch.go b/whatsapp_batch.go index 08dfb6a..7f43a3c 100644 --- a/whatsapp_batch.go +++ b/whatsapp_batch.go @@ -40,7 +40,8 @@ func (b WhatsAppBatch) Turn() bool { // The first line starts with [WhatsApp] and names the chat, its ID, and the day // of the first message. Each message has a line with its time and its sender, // and the messages of the user have the sender You. A batch that starts a turn -// ends with the line about the empty reply. +// ends with the line about the empty reply. Image markers start on their own +// lines so that the shared image reader can find them. func (b WhatsAppBatch) Text(location *time.Location) string { kind := "private chat" if b.Chat.Group { @@ -59,7 +60,12 @@ func (b WhatsAppBatch) Text(location *time.Location) string { sender += " (replies to you)" } - lines = append(lines, message.Time.In(location).Format("15:04")+" "+sender+": "+message.Text) + prefix := message.Time.In(location).Format("15:04") + " " + sender + ":" + if len(imageFiles(message.Text)) > 0 { + lines = append(lines, prefix, message.Text) + } else { + lines = append(lines, prefix+" "+message.Text) + } } if b.Turn() { @@ -76,10 +82,10 @@ type WhatsAppChat struct { Group bool `json:"group,omitempty"` } -// WhatsAppMessage is one text message of a batch: its ID, the name of its -// sender, its WhatsApp time, and its text. Mine marks a message of the user. -// Mention and Reply mark a message that mentions the user or replies to a -// message of the user. +// WhatsAppMessage is one message of a batch: its ID, the name of its sender, +// its WhatsApp time, and its text. Mine marks a message of the user. Mention +// and Reply mark a message that mentions the user or replies to a message of +// the user. type WhatsAppMessage struct { ID string `json:"id"` Sender string `json:"sender"` diff --git a/whatsapp_fake_helper_test.go b/whatsapp_fake_helper_test.go index d8bb9bb..e936f45 100644 --- a/whatsapp_fake_helper_test.go +++ b/whatsapp_fake_helper_test.go @@ -83,6 +83,10 @@ type WhatsAppFake struct { logoutErr error groups []*types.GroupInfo sendErr error + downloadData []byte + downloadErr error + downloadWait time.Duration + downloads []whatsmeow.DownloadableMessage phones []string logouts int sent []WhatsAppFakeSend @@ -168,6 +172,36 @@ func (f *WhatsAppFake) GetJoinedGroups(_ context.Context) ([]*types.GroupInfo, e return f.groups, nil } +// Download waits for downloadWait, records the request, and returns the made-up +// image or downloadErr. A canceled context ends the wait. +func (f *WhatsAppFake) Download(ctx context.Context, message whatsmeow.DownloadableMessage) ([]byte, error) { + if f.downloadWait > 0 { + timer := time.NewTimer(f.downloadWait) + defer timer.Stop() + + select { + case <-ctx.Done(): + return nil, ctx.Err() + case <-timer.C: + } + } + + f.mutex.Lock() + defer f.mutex.Unlock() + + f.downloads = append(f.downloads, message) + + return slices.Clone(f.downloadData), f.downloadErr +} + +// Downloads returns the media requested from the fake. +func (f *WhatsAppFake) Downloads() []whatsmeow.DownloadableMessage { + f.mutex.Lock() + defer f.mutex.Unlock() + + return slices.Clone(f.downloads) +} + // GenerateMessageID returns a new message ID. func (f *WhatsAppFake) GenerateMessageID() types.MessageID { f.mutex.Lock() diff --git a/whatsapp_image.go b/whatsapp_image.go new file mode 100644 index 0000000..4277b0a --- /dev/null +++ b/whatsapp_image.go @@ -0,0 +1,65 @@ +package main + +import ( + "context" + "errors" + "fmt" + "time" + + "go.mau.fi/whatsmeow" + "go.mau.fi/whatsmeow/proto/waE2E" +) + +// whatsappImageTime is the longest time of an image download. +const whatsappImageTime = 2 * time.Minute + +// whatsappImage downloads only supported images and stores them before the +// event can be acknowledged. Both the declared size and the downloaded bytes +// must fit the image store. Decoded bytes determine the stored format. +func whatsappImage(ctx context.Context, client WhatsAppClient, store *ImageStore, message *waE2E.ImageMessage) (string, error) { + if message.GetFileLength() > imageMaxBytes { + return "", errImageSize + } + + switch message.GetMimetype() { + case "image/jpeg", "image/png", "image/webp": + default: + return "", errImageFormat + } + + if client == nil { + return "", errors.New("WhatsApp client is not available") + } + + data, err := client.Download(ctx, message) + if err != nil { + return "", fmt.Errorf("download image: %w", whatsappImageDownloadError(err)) + } + + name, err := store.Save(data) + if err != nil { + return "", fmt.Errorf("store image: %w", err) + } + + return name, nil +} + +// whatsappImageDownloadError keeps only known safe causes. Library errors can +// contain signed URLs or direct paths even when request preparation fails, so +// arbitrary error text cannot go to the daemon log. +func whatsappImageDownloadError(err error) error { + switch { + case errors.Is(err, context.Canceled): + return context.Canceled + case errors.Is(err, context.DeadlineExceeded): + return context.DeadlineExceeded + } + + var response whatsmeow.DownloadHTTPError + + if errors.As(err, &response) && response.Response != nil { + return fmt.Errorf("media server returned HTTP %d", response.StatusCode) + } + + return errors.New("media download failed") +} diff --git a/whatsapp_image_test.go b/whatsapp_image_test.go new file mode 100644 index 0000000..f00bc3a --- /dev/null +++ b/whatsapp_image_test.go @@ -0,0 +1,586 @@ +package main + +import ( + "bytes" + "context" + "encoding/base64" + "errors" + "fmt" + "image" + "image/gif" + "image/jpeg" + "net/http" + "net/url" + "os" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/stretchr/testify/require" + "go.mau.fi/whatsmeow" + "go.mau.fi/whatsmeow/proto/waE2E" + "go.mau.fi/whatsmeow/types" + "go.mau.fi/whatsmeow/types/events" +) + +func TestWhatsAppImagesAreDurableBeforeAcknowledgement(t *testing.T) { + testCases := []struct { + name string + sender types.JID + }{ + { + name: "a contact image keeps its caption and pixels", + sender: whatsappTestAna, + }, + { + name: "an own image keeps its caption and pixels", + sender: whatsappTestOwn, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + dir := t.TempDir() + whatsappTestSession(t, dir, true) + + whatsapp, fake, logs := whatsappStartTest(t, dir, newTestClock()) + whatsapp.client = fake + data := imageTestPNG(t, 3, 2) + fake.downloadData = data + event := whatsappImageTestEvent(tc.sender, "Read this label.") + require.True(t, whatsapp.handle(event)) + require.Len(t, fake.Downloads(), 1) + require.Same(t, event.Message.ImageMessage, fake.Downloads()[0]) + + restored := whatsappTestState(t, filepath.Join(whatsapp.dir, whatsappStateFileName)) + require.NoError(t, restored.CloseAll()) + require.Len(t, restored.Closed(), 1) + + batch := restored.Closed()[0] + names := imageFiles(batch.Messages[0].Text) + require.Len(t, names, 1) + require.Equal(t, tc.sender == whatsappTestOwn, batch.Messages[0].Mine) + require.Equal(t, imageText(names, "Read this label."), batch.Messages[0].Text) + require.Equal(t, names, imageFiles(batch.Text(time.UTC))) + + store := &ImageStore{dir: filepath.Join(dir, imageDirName)} + saved, err := store.Read(names[0]) + require.NoError(t, err) + require.Equal(t, data, saved) + + info, err := os.Stat(filepath.Join(store.dir, names[0])) + require.NoError(t, err) + require.Equal(t, os.FileMode(0o600), info.Mode().Perm()) + require.NotContains(t, logs.String(), "Read this label") + require.NotContains(t, logs.String(), "base64") + }) + } +} + +func TestWhatsAppImageDownloadCanOutlastNameLookup(t *testing.T) { + whatsapp, fake := whatsappStartTestLinked(t) + whatsapp.client = fake + whatsapp.inbox.lookupTime = 10 * time.Millisecond + whatsapp.inbox.imageTime = time.Second + fake.downloadWait = 50 * time.Millisecond + data := imageTestPNG(t, 3, 2) + fake.downloadData = data + + require.True(t, whatsapp.handle(whatsappImageTestEvent(whatsappTestAna, "Slow photo."))) + + restored := whatsappTestState(t, whatsapp.state.path) + require.NoError(t, restored.CloseAll()) + require.Len(t, restored.Closed(), 1) + + text := restored.Closed()[0].Messages[0].Text + names := imageFiles(text) + require.Len(t, names, 1) + require.Equal(t, imageText(names, "Slow photo."), text) + + saved, err := whatsapp.inbox.images.Read(names[0]) + require.NoError(t, err) + require.Equal(t, data, saved) +} + +func TestWhatsAppImageFailureKeepsTheBatchMoving(t *testing.T) { + dir := t.TempDir() + whatsappTestSession(t, dir, true) + + whatsapp, fake, logs := whatsappStartTest(t, dir, newTestClock()) + whatsapp.client = fake + fake.downloadErr = errors.New("download failed for " + whatsappTestNumber + " and " + whatsappTestAna.User) + event := whatsappImageTestEvent(whatsappTestAna, "Keep this caption.") + require.True(t, whatsapp.handle(event)) + require.True(t, whatsapp.handle(whatsappTestMessage(whatsappTestAna, whatsappTestAna, "next", "Next text.", event.Info.Timestamp))) + + restored := whatsappTestState(t, filepath.Join(whatsapp.dir, whatsappStateFileName)) + require.NoError(t, restored.CloseAll()) + require.Len(t, restored.Closed(), 1) + + messages := restored.Closed()[0].Messages + require.Len(t, messages, 2) + require.Equal(t, "(image not downloaded)\nKeep this caption.", messages[0].Text) + require.Equal(t, "Next text.", messages[1].Text) + require.Empty(t, imageFiles(restored.Closed()[0].Text(time.UTC))) + require.Contains(t, logs.String(), "<3>image receive failed chat=") + require.Contains(t, logs.String(), ": download image: media download failed") + require.NotContains(t, logs.String(), whatsappTestNumber) + require.NotContains(t, logs.String(), whatsappTestAna.User) + require.NotContains(t, logs.String(), "Keep this caption") + require.NotContains(t, logs.String(), "base64") +} + +func TestWhatsAppImageDownloadErrorsKeepOnlySafeCauses(t *testing.T) { + mediaURL := "https://example.com/media/secret-path?auth=secret-token&hash=secret-hash" + directPath := "media/secret-path?auth=secret-token&hash=secret-hash" + testCases := []struct { + name string + err error + cause string + }{ + { + name: "signed request URLs do not reach the log", + err: &url.Error{ + Op: "Get", + URL: mediaURL, + Err: errors.New("connection failed"), + }, + cause: "media download failed", + }, + { + name: "invalid direct paths do not reach the log", + err: fmt.Errorf("media download path does not start with slash: %s", directPath), + cause: "media download failed", + }, + { + name: "request preparation errors do not expose media paths", + err: fmt.Errorf("failed to prepare request: parse %q: invalid control character", mediaURL), + cause: "media download failed", + }, + { + name: "cancellation keeps its safe cause without the URL", + err: fmt.Errorf("request %s: %w", mediaURL, context.Canceled), + cause: "context canceled", + }, + { + name: "a deadline keeps its safe cause without the path", + err: fmt.Errorf("request %s: %w", directPath, context.DeadlineExceeded), + cause: "context deadline exceeded", + }, + { + name: "HTTP failures keep only the status code", + err: whatsmeow.DownloadHTTPError{Response: &http.Response{ + StatusCode: http.StatusForbidden, + Status: "403 " + mediaURL, + }}, + cause: "media server returned HTTP 403", + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + dir := t.TempDir() + whatsappTestSession(t, dir, true) + + whatsapp, fake, logs := whatsappStartTest(t, dir, newTestClock()) + whatsapp.client = fake + fake.downloadErr = tc.err + + require.True(t, whatsapp.handle(whatsappImageTestEvent(whatsappTestAna, "Keep this caption."))) + require.NoError(t, whatsapp.state.CloseAll()) + require.Equal(t, "(image not downloaded)\nKeep this caption.", whatsapp.state.Closed()[0].Messages[0].Text) + require.Contains(t, logs.String(), "<3>image receive failed chat=") + require.Contains(t, logs.String(), ": download image: "+tc.cause) + + for _, secret := range []string{mediaURL, directPath, "example.com", "secret-path", "secret-token", "secret-hash", whatsappTestAna.User, "Keep this caption."} { + require.NotContains(t, logs.String(), secret) + } + }) + } +} + +func TestWhatsAppImageLimitsKeepMarkersWithoutStoredPixels(t *testing.T) { + var gifBytes bytes.Buffer + + require.NoError(t, gif.Encode(&gifBytes, image.NewRGBA(image.Rect(0, 0, 2, 2)), nil)) + + testCases := []struct { + name string + mime string + size uint64 + data []byte + downloads int + }{ + { + name: "declared large images are not downloaded", + mime: "image/png", + size: imageMaxBytes + 1, + }, + { + name: "large downloaded images become markers", + mime: "image/png", + data: bytes.Repeat([]byte{0}, imageMaxBytes+1), + downloads: 1, + }, + { + name: "unsupported image types are not downloaded", + mime: "image/gif", + }, + { + name: "GIF bytes cannot claim to be PNG", + mime: "image/png", + data: gifBytes.Bytes(), + downloads: 1, + }, + { + name: "invalid image bytes become markers", + mime: "image/png", + data: []byte("not an image"), + downloads: 1, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + whatsapp, fake := whatsappStartTestLinked(t) + whatsapp.client = fake + fake.downloadData = tc.data + event := whatsappImageTestEvent(whatsappTestAna, "") + event.Message.ImageMessage.Mimetype = new(tc.mime) + event.Message.ImageMessage.FileLength = new(tc.size) + require.True(t, whatsapp.handle(event)) + require.Len(t, fake.Downloads(), tc.downloads) + require.NoError(t, whatsapp.state.CloseAll()) + require.Len(t, whatsapp.state.Closed(), 1) + require.Equal(t, "(image not downloaded)", whatsapp.state.Closed()[0].Messages[0].Text) + + _, err := os.Stat(whatsapp.inbox.images.dir) + require.ErrorIs(t, err, os.ErrNotExist) + }) + } +} + +func TestWhatsAppSupportsJPEGPNGAndWebPBytes(t *testing.T) { + var jpegBytes bytes.Buffer + + require.NoError(t, jpeg.Encode(&jpegBytes, image.NewRGBA(image.Rect(0, 0, 2, 2)), nil)) + + testCases := []struct { + name string + mime string + data []byte + suffix string + }{ + { + name: "PNG bytes keep the PNG extension", + mime: "image/png", + data: imageTestPNG(t, 2, 2), + suffix: ".png", + }, + { + name: "JPEG bytes set the JPEG extension even with a PNG label", + mime: "image/png", + data: jpegBytes.Bytes(), + suffix: ".jpg", + }, + { + name: "WebP bytes keep the WebP extension", + mime: "image/webp", + data: []byte{'R', 'I', 'F', 'F', 22, 0, 0, 0, 'W', 'E', 'B', 'P', 'V', 'P', '8', 'L', 10, 0, 0, 0, 0x2f, 0, 0, 0, 0, 0x07, 0x10, 0x11, 0x11, 0x88}, + suffix: ".webp", + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + whatsapp, fake := whatsappStartTestLinked(t) + whatsapp.client = fake + fake.downloadData = tc.data + event := whatsappImageTestEvent(whatsappTestAna, "") + event.Message.ImageMessage.Mimetype = new(tc.mime) + require.True(t, whatsapp.handle(event)) + require.NoError(t, whatsapp.state.CloseAll()) + + names := imageFiles(whatsapp.state.Closed()[0].Text(time.UTC)) + require.Len(t, names, 1) + require.True(t, strings.HasSuffix(names[0], tc.suffix)) + + data, err := whatsapp.inbox.images.Read(names[0]) + require.NoError(t, err) + require.Equal(t, tc.data, data) + }) + } +} + +func TestWhatsAppMediaMarkersNeverDownload(t *testing.T) { + testCases := []struct { + name string + message *waE2E.Message + text string + }{ + { + name: "stickers keep a text marker", + message: &waE2E.Message{ + StickerMessage: &waE2E.StickerMessage{}, + }, + text: "(sticker)", + }, + { + name: "videos keep a text marker and caption", + message: &waE2E.Message{ + VideoMessage: &waE2E.VideoMessage{Caption: new("Watch this.")}, + }, + text: "(video)\nWatch this.", + }, + { + name: "GIF playback keeps a text marker", + message: &waE2E.Message{ + VideoMessage: &waE2E.VideoMessage{GifPlayback: new(true)}, + }, + text: "(GIF)", + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + whatsapp, fake := whatsappStartTestLinked(t) + whatsapp.client = fake + event := whatsappImageTestEvent(whatsappTestAna, "") + event.Message = tc.message + require.True(t, whatsapp.handle(event)) + require.NoError(t, whatsapp.state.CloseAll()) + require.Len(t, whatsapp.state.Closed(), 1) + require.Equal(t, tc.text, whatsapp.state.Closed()[0].Messages[0].Text) + require.Empty(t, fake.Downloads()) + }) + } +} + +func TestWhatsAppPrivateImagesAreNotDownloaded(t *testing.T) { + testCases := []struct { + name string + change func(event *events.Message) + }{ + { + name: "view-once events are not read", + change: func(event *events.Message) { + event.IsViewOnce = true + }, + }, + { + name: "view-once image fields are not read", + change: func(event *events.Message) { + event.Message.ImageMessage.ViewOnce = new(true) + }, + }, + { + name: "disappearing images are not read", + change: func(event *events.Message) { + event.IsEphemeral = true + }, + }, + { + name: "self-chat images are not read", + change: func(event *events.Message) { + event.Info.Chat = whatsappTestOwn + }, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + whatsapp, fake := whatsappStartTestLinked(t) + whatsapp.client = fake + event := whatsappImageTestEvent(whatsappTestAna, "Secret caption.") + tc.change(event) + require.True(t, whatsapp.handle(event)) + require.NoError(t, whatsapp.state.CloseAll()) + require.Empty(t, whatsapp.state.Closed()) + require.Empty(t, fake.Downloads()) + }) + } +} + +func TestWhatsAppImageStateFailureDoesNotAcknowledge(t *testing.T) { + whatsapp, fake := whatsappStartTestLinked(t) + whatsapp.client = fake + fake.downloadData = imageTestPNG(t, 2, 2) + require.NoError(t, os.Mkdir(whatsapp.state.path, 0o700)) + require.False(t, whatsapp.handle(whatsappImageTestEvent(whatsappTestAna, "Keep this."))) + require.ErrorContains(t, <-whatsapp.inbox.failures, "save WhatsApp state") +} + +func TestWhatsAppImagesReachTheirRootOrRoutedChild(t *testing.T) { + testCases := []struct { + name string + target int + group bool + }{ + { + name: "private images wake the root with pixels", + }, + { + name: "routed group images wake only their child with pixels", + target: 1, + group: true, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + dir := t.TempDir() + server := newModelFake(t) + server.reply = whatsappImageTestReply + test := newDaemonMessageTest(t, dir, server) + test.agents[0].agent.config.Model = "gpt-test-root" + + whatsappTestSession(t, dir, true) + + whatsapp, fake, logs := whatsappStartTest(t, dir, newTestClock()) + whatsapp.client = fake + data := imageTestPNG(t, 3, 2) + fake.downloadData = data + event := whatsappImageTestEvent(whatsappTestAna, "What do you see?") + if tc.group { + event.Info.Chat = whatsappTestFamily + event.Info.IsGroup = true + + whatsapp.groups.set(whatsappTestFamily.String(), "Garden plans") + require.NoError(t, test.messages.routes.Set("whatsapp:"+whatsappTestFamily.String(), "Garden")) + } + + test.Start(t, tc.target) + require.True(t, whatsapp.handle(event)) + require.NoError(t, whatsapp.inbox.deliver(t.Context(), test.loops[0], whatsapp.link(), true)) + test.events[tc.target].WaitFor(t, "idle", 1) + + text := test.events[tc.target].Last("work") + names := imageFiles(text) + require.Len(t, names, 1) + require.Contains(t, text, "[image "+names[0]+"]\nWhat do you see?") + require.Equal(t, 1, test.agents[tc.target].memory.Len()) + require.Empty(t, whatsapp.state.Closed()) + + if tc.target != 0 { + require.Zero(t, test.agents[0].memory.Len()) + require.False(t, test.loops[0].waiting()) + } + + model := test.agents[tc.target].agent.config.Model + turnCalls := 0 + for index := range server.Calls() { + call := server.Call(index) + if call.body["model"] == model { + turnCalls++ + + require.Contains(t, string(call.raw), "data:image/png;base64,"+base64.StdEncoding.EncodeToString(data)) + } + } + + require.Equal(t, 1, turnCalls) + require.NotContains(t, logs.String()+test.logs.String(), "What do you see?") + require.NotContains(t, logs.String()+test.logs.String(), "base64") + }) + } +} + +func TestWhatsAppImageRoutingSurvivesRestartWithoutDuplicateEntries(t *testing.T) { + dir := t.TempDir() + server := newModelFake(t) + server.reply = whatsappImageTestReply + first := newDaemonMessageTest(t, dir, server) + whatsappTestSession(t, dir, true) + + whatsapp, fake, _ := whatsappStartTest(t, dir, newTestClock()) + whatsapp.client = fake + fake.downloadData = imageTestPNG(t, 3, 2) + event := whatsappImageTestEvent(whatsappTestAna, "Keep this image.") + require.NoError(t, first.messages.routes.Set("whatsapp:"+whatsappTestAna.String(), "Garden")) + require.True(t, whatsapp.handle(event)) + require.NoError(t, whatsapp.state.CloseAll()) + + batch := whatsapp.state.Closed()[0] + text := batch.Text(time.UTC) + routed, err := first.messages.RouteWhatsApp(batch, text) + require.NoError(t, err) + require.True(t, routed) + first.Close(t) + + second := newDaemonMessageTest(t, dir, server) + restored := whatsappRouteTestInbox(t, dir) + require.NoError(t, restored.deliver(t.Context(), second.loops[0], WhatsAppLink{}, true)) + require.Empty(t, restored.state.Closed()) + require.Zero(t, second.agents[0].memory.Len()) + require.Equal(t, 1, second.agents[1].memory.Len()) + require.True(t, second.loops[1].waiting()) + + message, err := second.agents[1].memory.Message(0) + require.NoError(t, err) + require.Equal(t, text, message.Text) + require.Len(t, imageFiles(message.Text), 1) + second.Start(t, 1) + second.events[1].WaitFor(t, "idle", 1) + require.Empty(t, second.Queue(t).Pending) + + turnCalls := 0 + for index := range server.Calls() { + call := server.Call(index) + if call.body["model"] == "gpt-test-garden" { + turnCalls++ + + require.Contains(t, string(call.raw), "input_image") + } + } + + require.Equal(t, 1, turnCalls) + second.Close(t) + + third := newDaemonMessageTest(t, dir, server) + require.Equal(t, 1, third.agents[1].memory.Len()) + require.False(t, third.loops[1].waiting()) +} + +func TestWhatsAppImageMentionsAndRepliesUseImageContext(t *testing.T) { + whatsapp, fake := whatsappStartTestLinked(t) + whatsapp.client = fake + fake.downloadData = imageTestPNG(t, 2, 2) + event := whatsappImageTestEvent(whatsappTestAna, "@"+whatsappTestLID+" check "+whatsappTestNumber) + event.Info.Chat = whatsappTestFamily + event.Message.ImageMessage.ContextInfo = &waE2E.ContextInfo{ + MentionedJID: []string{whatsappTestLID + "@lid"}, + Participant: new(whatsappTestOwn.String()), + } + + require.True(t, whatsapp.handle(event)) + require.NoError(t, whatsapp.state.CloseAll()) + + batch := whatsapp.state.Closed()[0] + require.True(t, batch.Turn()) + require.True(t, batch.Messages[0].Mention) + require.True(t, batch.Messages[0].Reply) + require.Contains(t, batch.Text(time.UTC), "@you check OWN") + require.NotContains(t, batch.Text(time.UTC), whatsappTestNumber) + require.NotContains(t, batch.Text(time.UTC), whatsappTestLID) +} + +func whatsappImageTestEvent(sender types.JID, caption string) *events.Message { + event := whatsappTestMessage(whatsappTestAna, sender, "image", "", time.Date(2026, 1, 1, 12, 0, 0, 0, time.UTC)) + event.Message = &waE2E.Message{ + ImageMessage: &waE2E.ImageMessage{ + Mimetype: new("image/png"), + Caption: new(caption), + }, + } + + return event +} + +func whatsappImageTestReply(w http.ResponseWriter, call ModelFakeCall) { + text := "A red test image." + if call.body["model"] == "gpt-test-root" || call.body["model"] == "gpt-test-garden" { + text = "" + } + + modelWriteEvents(w, modelCompletedEvent([]any{modelMessageItem(text)}, 100, 0, 5)) +} diff --git a/whatsapp_inbox.go b/whatsapp_inbox.go index adff79e..5121976 100644 --- a/whatsapp_inbox.go +++ b/whatsapp_inbox.go @@ -2,7 +2,9 @@ package main import ( "context" + "crypto/sha256" "fmt" + "log/slog" "slices" "strings" "sync" @@ -23,19 +25,24 @@ const ( ) // newWhatsAppInbox returns the inbox of the batches in state, with clock for -// the times of the batch texts and groups for the names of the groups. -func newWhatsAppInbox(state *WhatsAppState, clock func() time.Time, groups *WhatsAppGroups) *WhatsAppInbox { +// the times of the batch texts and groups for the names of the groups. images +// holds shared images, and logger gets image failures. +func newWhatsAppInbox(state *WhatsAppState, clock func() time.Time, groups *WhatsAppGroups, images *ImageStore, logger *slog.Logger) *WhatsAppInbox { return &WhatsAppInbox{ state: state, clock: clock, groups: groups, + images: images, + logger: logger, + lookupTime: whatsappLookupTime, + imageTime: whatsappImageTime, catchupLimit: whatsappCatchupLimit, wake: make(chan struct{}, 1), failures: make(chan error, 1), } } -// WhatsAppInbox puts the text messages into batches, one chat at a time, as +// WhatsAppInbox puts messages into batches, one chat at a time, as // WhatsAppState says. deliver logs each closed batch as a work message that // starts with [WhatsApp], and a batch that needs the user wakes the turn loop. // @@ -61,6 +68,10 @@ type WhatsAppInbox struct { state *WhatsAppState clock func() time.Time groups *WhatsAppGroups + images *ImageStore + logger *slog.Logger + lookupTime time.Duration + imageTime time.Duration catchupLimit time.Duration wake chan struct{} failures chan error @@ -212,13 +223,13 @@ func (i *WhatsAppInbox) startCatchup() { }) } -// receive puts the text of event into its batch and wakes Run. link gives the -// device and the own IDs. It returns false when the state file cannot hold the +// receive stores an image before its batch and wakes Run. link gives the device +// and the own IDs. It returns false when the state file cannot hold the // message, so that the library does not acknowledge it, and Run stops the -// daemon. A message that is not text, or that the batches do not take, is -// handled with nothing to do. +// daemon. An image that cannot be downloaded or stored gets a text marker. A +// message that the batches do not take is handled with nothing to do. func (i *WhatsAppInbox) receive(link WhatsAppLink, event *events.Message) bool { - ctx, cancel := context.WithTimeout(context.Background(), whatsappLookupTime) + ctx, cancel := context.WithTimeout(context.Background(), i.lookupTime) defer cancel() chat, message, found := whatsappMessage(ctx, link.device, link.own, event) @@ -226,6 +237,13 @@ func (i *WhatsAppInbox) receive(link WhatsAppLink, event *events.Message) bool { return true } + if event.Message.GetImageMessage() != nil { + ctx, cancel := context.WithTimeout(context.Background(), i.imageTime) + defer cancel() + + message.Text = i.imageText(ctx, link, event, message.Text) + } + _, err := i.state.Add(chat, message, time.Now()) if err != nil { select { @@ -244,6 +262,28 @@ func (i *WhatsAppInbox) receive(link WhatsAppLink, event *events.Message) bool { return true } +// imageText stores the image of event before returning its marker and caption. +// Failures keep the caption with a text marker. The daemon log identifies the +// chat by a short hash, not its phone number. +func (i *WhatsAppInbox) imageText(ctx context.Context, link WhatsAppLink, event *events.Message, caption string) string { + name, err := whatsappImage(ctx, link.client, i.images, event.Message.GetImageMessage()) + if err != nil { + chat := event.Info.Chat.ToNonAD() + digest := sha256.Sum256([]byte(chat.String())) + redact := append(slices.Clone(link.own), chat.User) + i.logger.Error("image receive failed", "chat", fmt.Sprintf("%x", digest[:6]), "err", whatsappRedact(redact, err.Error())) + + text := "(image not downloaded)" + if caption != "" { + text += "\n" + caption + } + + return text + } + + return imageText([]string{name}, caption) +} + // caughtUpBy ends the catch-up of the connection with the number catchups, when // no later connection started a new one. func (i *WhatsAppInbox) caughtUpBy(catchups int) { @@ -303,10 +343,10 @@ func (i *WhatsAppInbox) close() { // whatsappMessage returns the chat and the batch message of event, or false // when the batches do not take it. device is the linked device, and own holds -// the own IDs. The batches take the text of a private chat or a group, from -// another person or from the user. They do not take media, reactions, edits, -// deletions, view-once and disappearing messages, old history, status -// broadcasts, channels, and the chat of the user with itself. +// the own IDs. The batches take text, images, and media markers from a private +// chat or a group, from another person or from the user. They do not take +// reactions, edits, deletions, view-once and disappearing messages, old +// history, status broadcasts, channels, and the chat of the user with itself. // // A mention of the user becomes @you, another mention @NAME when the name is // known, and the own number and LID become OWN, so that they never go to the @@ -316,7 +356,7 @@ func whatsappMessage(ctx context.Context, device *store.Device, own []string, ev chatJID := info.Chat.ToNonAD() switch { case device == nil: - case event.IsEdit, info.Edit != "", event.IsEphemeral, event.IsViewOnce, event.SourceWebMsg != nil: + case event.IsEdit, info.Edit != "", event.IsEphemeral, event.IsViewOnce, event.Message.GetImageMessage().GetViewOnce(), event.Message.GetVideoMessage().GetViewOnce(), event.SourceWebMsg != nil: case chatJID.Server != types.DefaultUserServer && chatJID.Server != types.HiddenUserServer && chatJID.Server != types.GroupServer: case slices.Contains(own, chatJID.User): default: @@ -327,7 +367,7 @@ func whatsappMessage(ctx context.Context, device *store.Device, own []string, ev } // whatsappText returns the chat and the batch message of event, a message that -// the batches take when it has text. +// the batches take. An image returns its caption until receive stores it. func whatsappText(ctx context.Context, device *store.Device, own []string, event *events.Message) (WhatsAppChat, WhatsAppMessage, bool) { info := event.Info text := event.Message.GetConversation() @@ -336,11 +376,33 @@ func whatsappText(ctx context.Context, device *store.Device, own []string, event text = extended.GetText() } - if strings.TrimSpace(text) == "" { - return WhatsAppChat{}, WhatsAppMessage{}, false + contextInfo := extended.GetContextInfo() + switch { + case event.Message.GetImageMessage() != nil: + image := event.Message.GetImageMessage() + text = image.GetCaption() + contextInfo = image.GetContextInfo() + case event.Message.GetStickerMessage() != nil: + text = "(sticker)" + contextInfo = event.Message.GetStickerMessage().GetContextInfo() + case event.Message.GetVideoMessage() != nil: + video := event.Message.GetVideoMessage() + text = "(video)" + if video.GetGifPlayback() { + text = "(GIF)" + } + + if video.GetCaption() != "" { + text += "\n" + video.GetCaption() + } + + contextInfo = video.GetContextInfo() + default: + if strings.TrimSpace(text) == "" { + return WhatsAppChat{}, WhatsAppMessage{}, false + } } - contextInfo := extended.GetContextInfo() message := WhatsAppMessage{ ID: info.ID, Time: info.Timestamp, diff --git a/whatsapp_inbox_test.go b/whatsapp_inbox_test.go index c3937df..cea65fe 100644 --- a/whatsapp_inbox_test.go +++ b/whatsapp_inbox_test.go @@ -133,7 +133,7 @@ func TestWhatsAppGroupTurns(t *testing.T) { } } -func TestWhatsAppTakesOnlyText(t *testing.T) { +func TestWhatsAppSkipsExcludedMessages(t *testing.T) { at := time.Date(2026, 1, 1, 12, 0, 0, 0, time.UTC) testCases := []struct { name string @@ -175,16 +175,6 @@ func TestWhatsAppTakesOnlyText(t *testing.T) { event.Info.Chat = types.StatusBroadcastJID }, }, - { - name: "a photo with a caption", - change: func(event *events.Message) { - event.Message = &waE2E.Message{ - ImageMessage: &waE2E.ImageMessage{ - Caption: new("look"), - }, - } - }, - }, { name: "a text of spaces", change: func(event *events.Message) { diff --git a/whatsapp_route_test.go b/whatsapp_route_test.go index 92c9cf3..9c855c9 100644 --- a/whatsapp_route_test.go +++ b/whatsapp_route_test.go @@ -393,7 +393,7 @@ func whatsappRouteTestInbox(t *testing.T, dir string) *WhatsAppInbox { state, err := openWhatsAppState(filepath.Join(dir, whatsappDirName, whatsappStateFileName), whatsappBatchQuiet, whatsappBatchWindow) require.NoError(t, err) - return newWhatsAppInbox(state, newTestClock().Now, newWhatsAppGroups(discardLog())) + return newWhatsAppInbox(state, newTestClock().Now, newWhatsAppGroups(discardLog()), &ImageStore{dir: filepath.Join(dir, imageDirName)}, discardLog()) } func whatsappRouteTestAdd(t *testing.T, inbox *WhatsAppInbox, chat WhatsAppChat, text string) {