From: Anderson A. Giacomolli Date: Tue, 10 Nov 2026 14:57:46 +0000 (-0300) Subject: Accept owner photos and image files in Telegram X-Git-Url: http://git.giacomolli.uk/onechat/commitdiff_plain/b2fc2d6a737dd5bab25e57ba822dd07258de93e6?ds=sidebyside Accept owner photos and image files in Telegram Download the largest supplied photo up to 1568 pixels, or an image file, into the shared store and submit its marker and exact caption. Refuse unsupported or large images with a short notice. Retry network failures without moving the offset, and stop on storage failures so images are not lost. Keep tokens and pixels out of daemon logs. --- diff --git a/README.rst b/README.rst index 8002ea0..cb47c98 100644 --- a/README.rst +++ b/README.rst @@ -157,8 +157,8 @@ replies show with Markdown styles and tables. These lines are commands: ``/whatsapp ...`` The WhatsApp commands above. -In Telegram, write as in the client. ``onechat help`` lists all -commands. +In Telegram, write as in the client, also with photos and image files. +``onechat help`` lists all commands. ********************* What the agent does @@ -190,6 +190,10 @@ WhatsApp ask. It proposes to route a chat that is only about one area to the child of that area, and sets the route after you agree. +Images + Send a photo, and the agent sees it. Its memory keeps what the image + shows, and it can look at an older image again. + Schedules Ask for reminders and timed tasks, such as "every weekday at 8, tell me what my groups talked about". Each run uses the quota, so runs are diff --git a/image.go b/image.go index d6a0b20..4b23f7c 100644 --- a/image.go +++ b/image.go @@ -22,6 +22,11 @@ const ( imageMaxBytes = 20 << 20 ) +var ( + errImageSize = errors.New("image exceeds 20 MB") + errImageFormat = errors.New("use a JPEG, PNG, or WebP image") +) + // ImageStore holds immutable images shared by the root, children, and // subagents. Names are the SHA-256 of the bytes with the decoded format as // extension. Publication is atomic and does not replace an existing image. @@ -97,12 +102,12 @@ func (s *ImageStore) Read(name string) ([]byte, error) { func imageFormat(data []byte) (string, error) { if len(data) > imageMaxBytes { - return "", errors.New("image exceeds 20 MB") + return "", errImageSize } config, format, err := image.DecodeConfig(bytes.NewReader(data)) if err != nil || config.Width <= 0 || config.Height <= 0 { - return "", errors.New("use a JPEG, PNG, or WebP image") + return "", errImageFormat } switch format { @@ -111,7 +116,7 @@ func imageFormat(data []byte) (string, error) { case "png", "webp": return format, nil default: - return "", errors.New("use a JPEG, PNG, or WebP image") + return "", errImageFormat } } @@ -136,7 +141,7 @@ func imageReadFile(path string) (data []byte, err error) { } if len(data) > imageMaxBytes { - return nil, errors.New("image exceeds 20 MB") + return nil, errImageSize } return data, nil @@ -162,13 +167,14 @@ func imageFiles(text string) []string { } func imageText(names []string, caption string) string { - var text strings.Builder - + lines := make([]string, 0, len(names)+1) for _, name := range names { - text.WriteString("[image " + name + "]\n") + lines = append(lines, "[image "+name+"]") } - text.WriteString(caption) + if caption != "" { + lines = append(lines, caption) + } - return strings.TrimSuffix(text.String(), "\n") + return strings.Join(lines, "\n") } diff --git a/telegram.go b/telegram.go index f2c687a..a595b68 100644 --- a/telegram.go +++ b/telegram.go @@ -133,13 +133,13 @@ func (c TelegramConfig) validate() error { } // TelegramBot connects the private chat of the owner with the agent. It polls -// the Bot API for new messages and sends each text message of the owner to the -// agent. It ignores every other update. Its sender sends the replies, the -// errors, the notices of the bot, and the user messages of the other clients to -// the chat, and shows the typing action while a turn runs. Only the replies -// have a format: their Markdown goes as rich messages, which Telegram renders -// itself, and as message entities only when Telegram refuses a rich message. -// The rest is plain text. Tool calls do not go to the chat. +// the Bot API for new messages and sends owner text and images to the agent. It +// ignores every other update. Its sender sends the replies, the errors, the +// notices of the bot, and the user messages of the other clients to the chat, +// and shows the typing action while a turn runs. Only the replies have a +// format: their Markdown goes as rich messages, which Telegram renders itself, +// and as message entities only when Telegram refuses a rich message. The rest +// is plain text. Tool calls do not go to the chat. // // The offset file holds the ID of the next update. It moves past an update // only after the message of the update is durable in the log, so a restart @@ -193,8 +193,8 @@ func (b *TelegramBot) Notify(event Event) { } // Run polls for the messages of the owner and sends the queued events to the -// chat until ctx ends. It returns an error only when a message or the offset -// cannot be written, because then the bot would lose or repeat messages. +// chat until ctx ends. It returns an error only when a message or the offset or +// image cannot be written, because then the bot would lose or repeat messages. func (b *TelegramBot) Run(ctx context.Context, l *TurnLoop, whatsapp *WhatsApp) error { ctx, cancel := context.WithCancel(ctx) defer cancel() @@ -258,9 +258,9 @@ func (b *TelegramBot) send(ctx context.Context) { } } -// poll gets the updates from the stored offset on and handles each one, until -// ctx ends. Each update is handled to its end also when ctx ends during it, so -// a stop does not repeat its message. +// poll gets updates from the stored offset and handles them until ctx ends. A +// failed image download retries from the same offset. A submitted message is +// flushed even if ctx ends, and only then can the offset advance. func (b *TelegramBot) poll(ctx context.Context, l *TurnLoop, whatsapp *WhatsApp) error { offset, err := telegramLoadOffset(b.offsetPath) if err != nil { @@ -273,7 +273,16 @@ func (b *TelegramBot) poll(ctx context.Context, l *TurnLoop, whatsapp *WhatsApp) back: "receive back", } + imageOutage := &TelegramOutage{ + logger: b.logger, + loss: "image receive failed", + back: "image receive back", + } + + imageWait := b.minWait wait := b.minWait + +poll: for ctx.Err() == nil { updates, err := b.api.GetUpdates(ctx, offset) if ctx.Err() != nil { @@ -290,16 +299,27 @@ func (b *TelegramBot) poll(ctx context.Context, l *TurnLoop, whatsapp *WhatsApp) outage.End() wait = b.minWait + for _, update := range updates { if ctx.Err() != nil { return nil } - err = b.handle(context.WithoutCancel(ctx), l, whatsapp, update) + err = b.handle(ctx, l, whatsapp, update) if err != nil { + retry, asked := telegramRetryable(err) + if retry { + imageWait = b.pause(ctx, imageOutage, err, imageWait, asked) + continue poll + } + return err } + imageOutage.End() + + imageWait = b.minWait + offset = update.UpdateID + 1 err = telegramSaveOffset(b.offsetPath, offset) if err != nil { @@ -373,9 +393,9 @@ func (b *TelegramBot) pause(ctx context.Context, outage *TelegramOutage, err err // to the agent, and handle returns when it is durable in the log. The line // /stop interrupts the turn, the line /stop NAME stops the subagent NAME, and a // line that starts with /whatsapp runs a WhatsApp command on whatsapp, whose -// answers go to the chat only. Any other message of the owner gets the answer -// that only text is supported. Every other update is ignored, and its content -// is not logged. +// answers go to the chat only. Photos and image documents enter the shared +// image store with their caption. Other media keep the text-only answer. Every +// other update is ignored, and its content is not logged. func (b *TelegramBot) handle(ctx context.Context, l *TurnLoop, whatsapp *WhatsApp, update TelegramUpdate) error { message := update.Message if message == nil || message.Chat.Type != "private" || message.Chat.ID != b.owner || message.From == nil || message.From.ID != b.owner { @@ -387,6 +407,41 @@ func (b *TelegramBot) handle(ctx context.Context, l *TurnLoop, whatsapp *WhatsAp name, stop := stopSubagentName(message.Text) args, isWhatsApp := whatsappCommandArgs(message.Text) switch { + case len(message.Photo) > 0 || message.Document != nil: + text, err := telegramImageText(ctx, b.api, l.agent.Images(), message) + if err != nil { + retry, _ := telegramRetryable(err) + if retry { + return fmt.Errorf("receive image: %w", err) + } + + switch { + case errors.Is(err, errTelegramOtherMedia): + b.notice(telegramTextOnly) + case errors.Is(err, errImageSize), errors.Is(err, errImageFormat), errors.Is(err, errTelegramPhotoSize): + b.notice(err.Error()) + default: + var answer *TelegramError + + if !errors.As(err, &answer) { + return fmt.Errorf("receive Telegram image: %w", err) + } + + b.notice("Cannot download image.") + } + + return nil + } + + _, err = l.Submit(telegramSource, text) + if err != nil { + return fmt.Errorf("submit Telegram image: %w", err) + } + + err = l.Flush(context.WithoutCancel(ctx)) + if err != nil { + return fmt.Errorf("flush Telegram image: %w", err) + } case message.Text == "": b.notice(telegramTextOnly) case strings.TrimSpace(message.Text) == stopCommand: @@ -410,7 +465,7 @@ func (b *TelegramBot) handle(ctx context.Context, l *TurnLoop, whatsapp *WhatsAp return err } - err = l.Flush(ctx) + err = l.Flush(context.WithoutCancel(ctx)) if err != nil { return fmt.Errorf("flush the message from Telegram: %w", err) } diff --git a/telegram_api.go b/telegram_api.go index 07d2bf1..32ad9a9 100644 --- a/telegram_api.go +++ b/telegram_api.go @@ -9,6 +9,7 @@ import ( "io" "net/http" "net/url" + "path" "strings" "time" ) @@ -102,6 +103,66 @@ func (b *TelegramAPI) GetUpdates(ctx context.Context, offset int64) ([]TelegramU return updates, err } +// GetImage downloads a file through the authenticated file endpoint. Errors +// never include its token or URL, and the body is limited before storage. +func (b *TelegramAPI) GetImage(ctx context.Context, id string) (data []byte, err error) { + var file struct { + Path string `json:"file_path"` + Size int64 `json:"file_size"` + } + + err = b.call(ctx, b.sendTimeout, "getFile", map[string]any{"file_id": id}, &file) + if err != nil { + return nil, fmt.Errorf("get image file: %w", err) + } + + if file.Size > imageMaxBytes { + return nil, errImageSize + } + + if file.Path == "" || path.Clean(file.Path) != file.Path || strings.HasPrefix(file.Path, "/") || strings.HasPrefix(file.Path, "../") || strings.ContainsAny(file.Path, "?%#\\") { + return nil, errors.New("invalid image file path") + } + + ctx, cancel := context.WithTimeout(ctx, b.sendTimeout) + defer cancel() + + request, err := http.NewRequestWithContext(ctx, http.MethodGet, b.base+"/file/bot"+b.token+"/"+file.Path, nil) + if err != nil { + return nil, fmt.Errorf("make image request: %s", b.redact(err)) + } + + response, err := b.http.Do(request) + if err != nil { + return nil, b.networkError("image download failed", err, b.sendTimeout) + } + + defer func() { + closeErr := response.Body.Close() + if closeErr != nil { + err = errors.Join(err, b.networkError("close image download", closeErr, b.sendTimeout)) + } + }() + + if response.StatusCode != http.StatusOK { + return nil, &TelegramError{ + Status: response.StatusCode, + Description: http.StatusText(response.StatusCode), + } + } + + data, err = io.ReadAll(io.LimitReader(response.Body, imageMaxBytes+1)) + if err != nil { + return nil, b.networkError("image cut off", err, b.sendTimeout) + } + + if len(data) > imageMaxBytes { + return nil, errImageSize + } + + return data, nil +} + // SendMessage sends text to chat with the format of entities, with no parse // mode, so the text needs no escaping. With no entities, the text is plain. A // call that takes longer than telegramSendTimeout fails with a @@ -291,12 +352,29 @@ type TelegramAnswer struct { } `json:"parameters"` } -// TelegramMessage is a message to the bot. Text is empty for a message that is -// not text, such as a photo. +// TelegramMessage holds the text, caption, and image sizes of a bot message. type TelegramMessage struct { - Chat TelegramChat `json:"chat"` - From *TelegramUser `json:"from"` - Text string `json:"text"` + Chat TelegramChat `json:"chat"` + From *TelegramUser `json:"from"` + Text string `json:"text"` + Caption string `json:"caption"` + Photo []TelegramPhoto `json:"photo"` + Document *TelegramDocument `json:"document"` +} + +// TelegramPhoto is one supplied size of a photo. +type TelegramPhoto struct { + FileID string `json:"file_id"` + Width int `json:"width"` + Height int `json:"height"` + FileSize int64 `json:"file_size"` +} + +// TelegramDocument is a file sent without photo compression. +type TelegramDocument struct { + FileID string `json:"file_id"` + MimeType string `json:"mime_type"` + FileSize int64 `json:"file_size"` } // TelegramChat is the chat of a message. Type is private for a chat with one diff --git a/telegram_fake_helper_test.go b/telegram_fake_helper_test.go index 3056aa1..6a89054 100644 --- a/telegram_fake_helper_test.go +++ b/telegram_fake_helper_test.go @@ -45,6 +45,7 @@ type TelegramFake struct { stall func(call TelegramFakeCall) bool added chan struct{} stopped chan struct{} + images map[string][]byte } // Add makes update available to getUpdates. @@ -86,7 +87,36 @@ func (f *TelegramFake) Sent() []string { return texts } +func (f *TelegramFake) Image(id string, data []byte) { + f.mutex.Lock() + defer f.mutex.Unlock() + + if f.images == nil { + f.images = make(map[string][]byte) + } + + f.images[id] = data +} + func (f *TelegramFake) serve(w http.ResponseWriter, r *http.Request) { + id, file := strings.CutPrefix(r.URL.Path, "/file/bot"+telegramTestToken+"/") + if file { + f.mutex.Lock() + + data, exists := f.images[id] + f.mutex.Unlock() + + if !exists { + http.NotFound(w, r) + + return + } + + _, _ = w.Write(data) + + return + } + method, found := strings.CutPrefix(r.URL.Path, "/bot"+telegramTestToken+"/") if !found { http.Error(w, `{"ok":false,"error_code":404,"description":"Not Found"}`, http.StatusNotFound) @@ -146,6 +176,15 @@ func (f *TelegramFake) serve(w http.ResponseWriter, r *http.Request) { call.ok = true f.record(call) + if method == "getFile" { + _ = json.NewEncoder(w).Encode(map[string]any{ + "ok": true, + "result": map[string]any{"file_path": params["file_id"]}, + }) + + return + } + if method != "getUpdates" { _, _ = io.WriteString(w, `{"ok":true,"result":true}`) return diff --git a/telegram_image.go b/telegram_image.go new file mode 100644 index 0000000..9b73341 --- /dev/null +++ b/telegram_image.go @@ -0,0 +1,71 @@ +package main + +import ( + "context" + "errors" + "fmt" + "strings" +) + +var ( + errTelegramOtherMedia = errors.New("only text messages are supported") + errTelegramPhotoSize = errors.New("no photo size fits 1568 pixels") +) + +// telegramImageText selects a supplied photo size, or an image document, and +// stores it before returning marker text with the unchanged caption. +func telegramImageText(ctx context.Context, api *TelegramAPI, store *ImageStore, message *TelegramMessage) (string, error) { + var ( + id string + size int64 + ) + + if len(message.Photo) > 0 { + var area int + + for _, photo := range message.Photo { + if photo.Width <= 0 || photo.Height <= 0 || max(photo.Width, photo.Height) > 1568 { + continue + } + + if photo.Width*photo.Height > area { + id = photo.FileID + size = photo.FileSize + area = photo.Width * photo.Height + } + } + + if id == "" { + return "", errTelegramPhotoSize + } + } else if message.Document != nil { + document := message.Document + switch document.MimeType { + case "image/jpeg", "image/png", "image/webp": + id = document.FileID + size = document.FileSize + default: + if strings.HasPrefix(document.MimeType, "image/") { + return "", errImageFormat + } + + return "", errTelegramOtherMedia + } + } + + if size > imageMaxBytes { + return "", errImageSize + } + + data, err := api.GetImage(ctx, id) + if err != nil { + return "", fmt.Errorf("download image: %w", err) + } + + name, err := store.Save(data) + if err != nil { + return "", fmt.Errorf("store image: %w", err) + } + + return imageText([]string{name}, message.Caption), nil +} diff --git a/telegram_image_test.go b/telegram_image_test.go new file mode 100644 index 0000000..837a64e --- /dev/null +++ b/telegram_image_test.go @@ -0,0 +1,259 @@ +package main + +import ( + "bytes" + "image" + "image/jpeg" + "net/http" + "os" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/stretchr/testify/require" +) + +func TestTelegramSelectsLargestPhotoAtMost1568Pixels(t *testing.T) { + server := newModelFake(t) + test := newTestTurnLoop(t, server) + telegram := newTelegramFake(t) + bot, logs := telegramStartTestBot(t, telegram) + picture := imageTestPNG(t, 8, 6) + telegram.Image("selected", picture) + + message := telegramOwnerMessage("") + message.Caption = "Read this label.\n" + message.Photo = []TelegramPhoto{ + { + FileID: "small", + Width: 100, + Height: 200, + }, + { + FileID: "too-large", + Width: 1600, + Height: 1200, + }, + { + FileID: "selected", + Width: 1568, + Height: 1176, + }, + { + FileID: "medium", + Width: 800, + Height: 600, + }, + } + require.NoError(t, bot.handle(t.Context(), test.loop, nil, TelegramUpdate{ + UpdateID: 1, + Message: message, + })) + test.events.WaitFor(t, "idle", 1) + + files, err := os.ReadDir(test.agent.Images().dir) + require.NoError(t, err) + require.Len(t, files, 1) + + logged, err := test.memory.Message(0) + require.NoError(t, err) + require.Equal(t, imageText([]string{files[0].Name()}, message.Caption), logged.Text) + require.True(t, strings.HasSuffix(logged.Text, "\n")) + require.Contains(t, string(server.Call(0).raw), "input_image") + require.NotContains(t, logs.String(), "base64") + require.NotContains(t, logs.String(), "Read this label") + + for _, call := range telegram.Calls() { + if call.method == "getFile" { + require.Equal(t, "selected", call.params["file_id"]) + } + } +} + +func TestTelegramImageDocumentsUseContentFormatAndRejectOtherMedia(t *testing.T) { + telegram := newTelegramFake(t) + png := imageTestPNG(t, 2, 2) + + var jpegBytes bytes.Buffer + + require.NoError(t, jpeg.Encode(&jpegBytes, image.NewRGBA(image.Rect(0, 0, 2, 2)), nil)) + + // A made-up one-pixel lossless WebP image. + webp := []byte{'R', 'I', 'F', 'F', 22, 0, 0, 0, 'W', 'E', 'B', 'P', 'V', 'P', '8', 'L', 10, 0, 0, 0, 0x2f, 0, 0, 0, 0, 0x07, 0x10, 0x11, 0x11, 0x88} + + telegram.Image("png", png) + telegram.Image("jpeg", jpegBytes.Bytes()) + telegram.Image("webp", webp) + telegram.Image("invalid", []byte("not an image")) + telegram.Image("large", bytes.Repeat([]byte{0}, imageMaxBytes+1)) + + api := newTelegramAPI(telegram.URL, telegramTestToken) + + store := &ImageStore{dir: filepath.Join(t.TempDir(), imageDirName)} + testCases := []struct { + name string + document TelegramDocument + suffix string + failure string + }{ + { + name: "accepts PNG documents", + document: TelegramDocument{ + FileID: "png", + MimeType: "image/png", + }, + suffix: ".png]", + }, + { + name: "accepts JPEG documents", + document: TelegramDocument{ + FileID: "jpeg", + MimeType: "image/jpeg", + }, + suffix: ".jpg]", + }, + { + name: "accepts WebP documents", + document: TelegramDocument{ + FileID: "webp", + MimeType: "image/webp", + }, + suffix: ".webp]", + }, + { + name: "other documents keep text answer", + document: TelegramDocument{ + FileID: "pdf", + MimeType: "application/pdf", + }, + failure: "only text messages are supported", + }, + { + name: "other images refused", + document: TelegramDocument{ + FileID: "gif", + MimeType: "image/gif", + }, + failure: "JPEG, PNG, or WebP", + }, + { + name: "invalid image refused", + document: TelegramDocument{ + FileID: "invalid", + MimeType: "image/png", + }, + failure: "JPEG, PNG, or WebP", + }, + { + name: "large download refused", + document: TelegramDocument{ + FileID: "large", + MimeType: "image/png", + }, + failure: "exceeds 20 MB", + }, + { + name: "large metadata refused", + document: TelegramDocument{ + FileID: "not-downloaded", + MimeType: "image/png", + FileSize: imageMaxBytes + 1, + }, + failure: "exceeds 20 MB", + }, + } + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + text, err := telegramImageText(t.Context(), api, store, &TelegramMessage{Document: &tc.document}) + if tc.failure != "" { + require.ErrorContains(t, err, tc.failure) + + return + } + + require.NoError(t, err) + require.True(t, strings.HasSuffix(text, tc.suffix)) + }) + } +} + +func TestTelegramRetriesImageDownloadBeforeAdvancingOffset(t *testing.T) { + telegram := newTelegramFake(t) + telegram.Image("photo", imageTestPNG(t, 2, 2)) + + tries := 0 + telegram.fail = func(call TelegramFakeCall) (int, string) { + if call.method == "getFile" { + tries++ + if tries == 1 { + return http.StatusServiceUnavailable, "try again" + } + } + + return 0, "" + } + bot, logs := telegramStartTestBot(t, telegram) + bot.minWait = time.Millisecond + bot.maxWait = 5 * time.Millisecond + server := newModelFake(t) + socket := daemonStartTestWithBot(t, server, bot) + terminal := terminalStartTestClient(t, socket) + terminal.WaitFor(t, "[connected to OneChat]") + + message := telegramOwnerMessage("") + message.Photo = []TelegramPhoto{{ + FileID: "photo", + Width: 2, + Height: 2, + }} + telegram.Add(TelegramUpdate{ + UpdateID: 7, + Message: message, + }) + terminal.WaitFor(t, "[done]") + require.Eventually(t, func() bool { + offset, err := telegramLoadOffset(bot.offsetPath) + return err == nil && offset == 8 + }, 10*time.Second, time.Millisecond) + require.Contains(t, logs.String(), "receive failed") + require.Contains(t, logs.String(), "receive back") + require.NotContains(t, logs.String(), telegramTestToken) + require.NotContains(t, logs.String(), "base64") +} + +func TestTelegramRefusesPhotoWhenEverySizeIsTooLarge(t *testing.T) { + telegram := newTelegramFake(t) + _, err := telegramImageText(t.Context(), newTelegramAPI(telegram.URL, telegramTestToken), &ImageStore{dir: t.TempDir()}, &TelegramMessage{Photo: []TelegramPhoto{{ + FileID: "too-big", + Width: 1600, + Height: 1000, + }}}) + require.ErrorContains(t, err, "1568 pixels") + require.Empty(t, telegram.Calls()) +} + +func TestTelegramStorageFailureDoesNotAcceptTheImage(t *testing.T) { + telegram := newTelegramFake(t) + telegram.Image("photo", imageTestPNG(t, 2, 2)) + + test := newTestTurnLoop(t, newModelFake(t)) + require.NoError(t, os.WriteFile(test.agent.Images().dir, []byte("blocked"), 0o600)) + + bot, _ := telegramStartTestBot(t, telegram) + message := telegramOwnerMessage("") + message.Photo = []TelegramPhoto{{ + FileID: "photo", + Width: 2, + Height: 2, + }} + err := bot.handle(t.Context(), test.loop, nil, TelegramUpdate{ + UpdateID: 1, + Message: message, + }) + require.ErrorContains(t, err, "create image directory") + require.Equal(t, 0, test.memory.Len()) + + _, found := bot.next() + require.False(t, found) +} diff --git a/turn_test.go b/turn_test.go index 949c0b8..3b86f0c 100644 --- a/turn_test.go +++ b/turn_test.go @@ -388,7 +388,7 @@ func turnInputTexts(t *testing.T, call ModelFakeCall) []string { func newTestTurnLoopWith(t *testing.T, server *ModelFake, summarizer Summarizer) *TurnTestLoop { t.Helper() - agent, err := openAgent(t.TempDir()) + agent, err := openAgent(filepath.Join(t.TempDir(), "agent")) require.NoError(t, err) agent.config.Workdir = t.TempDir()