``/whatsapp ...``
The WhatsApp commands above.
-In Telegram, write as in the client. ``onechat help`` lists all
-commands.
+In Telegram, write as in the client, also with photos and image files.
+``onechat help`` lists all commands.
*********************
What the agent does
ask. It proposes to route a chat that is only about one area to the
child of that area, and sets the route after you agree.
+Images
+ Send a photo, and the agent sees it. Its memory keeps what the image
+ shows, and it can look at an older image again.
+
Schedules
Ask for reminders and timed tasks, such as "every weekday at 8, tell
me what my groups talked about". Each run uses the quota, so runs are
imageMaxBytes = 20 << 20
)
+var (
+ errImageSize = errors.New("image exceeds 20 MB")
+ errImageFormat = errors.New("use a JPEG, PNG, or WebP image")
+)
+
// ImageStore holds immutable images shared by the root, children, and
// subagents. Names are the SHA-256 of the bytes with the decoded format as
// extension. Publication is atomic and does not replace an existing image.
func imageFormat(data []byte) (string, error) {
if len(data) > imageMaxBytes {
- return "", errors.New("image exceeds 20 MB")
+ return "", errImageSize
}
config, format, err := image.DecodeConfig(bytes.NewReader(data))
if err != nil || config.Width <= 0 || config.Height <= 0 {
- return "", errors.New("use a JPEG, PNG, or WebP image")
+ return "", errImageFormat
}
switch format {
case "png", "webp":
return format, nil
default:
- return "", errors.New("use a JPEG, PNG, or WebP image")
+ return "", errImageFormat
}
}
}
if len(data) > imageMaxBytes {
- return nil, errors.New("image exceeds 20 MB")
+ return nil, errImageSize
}
return data, nil
}
func imageText(names []string, caption string) string {
- var text strings.Builder
-
+ lines := make([]string, 0, len(names)+1)
for _, name := range names {
- text.WriteString("[image " + name + "]\n")
+ lines = append(lines, "[image "+name+"]")
}
- text.WriteString(caption)
+ if caption != "" {
+ lines = append(lines, caption)
+ }
- return strings.TrimSuffix(text.String(), "\n")
+ return strings.Join(lines, "\n")
}
}
// TelegramBot connects the private chat of the owner with the agent. It polls
-// the Bot API for new messages and sends each text message of the owner to the
-// agent. It ignores every other update. Its sender sends the replies, the
-// errors, the notices of the bot, and the user messages of the other clients to
-// the chat, and shows the typing action while a turn runs. Only the replies
-// have a format: their Markdown goes as rich messages, which Telegram renders
-// itself, and as message entities only when Telegram refuses a rich message.
-// The rest is plain text. Tool calls do not go to the chat.
+// the Bot API for new messages and sends owner text and images to the agent. It
+// ignores every other update. Its sender sends the replies, the errors, the
+// notices of the bot, and the user messages of the other clients to the chat,
+// and shows the typing action while a turn runs. Only the replies have a
+// format: their Markdown goes as rich messages, which Telegram renders itself,
+// and as message entities only when Telegram refuses a rich message. The rest
+// is plain text. Tool calls do not go to the chat.
//
// The offset file holds the ID of the next update. It moves past an update
// only after the message of the update is durable in the log, so a restart
}
// Run polls for the messages of the owner and sends the queued events to the
-// chat until ctx ends. It returns an error only when a message or the offset
-// cannot be written, because then the bot would lose or repeat messages.
+// chat until ctx ends. It returns an error only when a message or the offset or
+// image cannot be written, because then the bot would lose or repeat messages.
func (b *TelegramBot) Run(ctx context.Context, l *TurnLoop, whatsapp *WhatsApp) error {
ctx, cancel := context.WithCancel(ctx)
defer cancel()
}
}
-// poll gets the updates from the stored offset on and handles each one, until
-// ctx ends. Each update is handled to its end also when ctx ends during it, so
-// a stop does not repeat its message.
+// poll gets updates from the stored offset and handles them until ctx ends. A
+// failed image download retries from the same offset. A submitted message is
+// flushed even if ctx ends, and only then can the offset advance.
func (b *TelegramBot) poll(ctx context.Context, l *TurnLoop, whatsapp *WhatsApp) error {
offset, err := telegramLoadOffset(b.offsetPath)
if err != nil {
back: "receive back",
}
+ imageOutage := &TelegramOutage{
+ logger: b.logger,
+ loss: "image receive failed",
+ back: "image receive back",
+ }
+
+ imageWait := b.minWait
wait := b.minWait
+
+poll:
for ctx.Err() == nil {
updates, err := b.api.GetUpdates(ctx, offset)
if ctx.Err() != nil {
outage.End()
wait = b.minWait
+
for _, update := range updates {
if ctx.Err() != nil {
return nil
}
- err = b.handle(context.WithoutCancel(ctx), l, whatsapp, update)
+ err = b.handle(ctx, l, whatsapp, update)
if err != nil {
+ retry, asked := telegramRetryable(err)
+ if retry {
+ imageWait = b.pause(ctx, imageOutage, err, imageWait, asked)
+ continue poll
+ }
+
return err
}
+ imageOutage.End()
+
+ imageWait = b.minWait
+
offset = update.UpdateID + 1
err = telegramSaveOffset(b.offsetPath, offset)
if err != nil {
// to the agent, and handle returns when it is durable in the log. The line
// /stop interrupts the turn, the line /stop NAME stops the subagent NAME, and a
// line that starts with /whatsapp runs a WhatsApp command on whatsapp, whose
-// answers go to the chat only. Any other message of the owner gets the answer
-// that only text is supported. Every other update is ignored, and its content
-// is not logged.
+// answers go to the chat only. Photos and image documents enter the shared
+// image store with their caption. Other media keep the text-only answer. Every
+// other update is ignored, and its content is not logged.
func (b *TelegramBot) handle(ctx context.Context, l *TurnLoop, whatsapp *WhatsApp, update TelegramUpdate) error {
message := update.Message
if message == nil || message.Chat.Type != "private" || message.Chat.ID != b.owner || message.From == nil || message.From.ID != b.owner {
name, stop := stopSubagentName(message.Text)
args, isWhatsApp := whatsappCommandArgs(message.Text)
switch {
+ case len(message.Photo) > 0 || message.Document != nil:
+ text, err := telegramImageText(ctx, b.api, l.agent.Images(), message)
+ if err != nil {
+ retry, _ := telegramRetryable(err)
+ if retry {
+ return fmt.Errorf("receive image: %w", err)
+ }
+
+ switch {
+ case errors.Is(err, errTelegramOtherMedia):
+ b.notice(telegramTextOnly)
+ case errors.Is(err, errImageSize), errors.Is(err, errImageFormat), errors.Is(err, errTelegramPhotoSize):
+ b.notice(err.Error())
+ default:
+ var answer *TelegramError
+
+ if !errors.As(err, &answer) {
+ return fmt.Errorf("receive Telegram image: %w", err)
+ }
+
+ b.notice("Cannot download image.")
+ }
+
+ return nil
+ }
+
+ _, err = l.Submit(telegramSource, text)
+ if err != nil {
+ return fmt.Errorf("submit Telegram image: %w", err)
+ }
+
+ err = l.Flush(context.WithoutCancel(ctx))
+ if err != nil {
+ return fmt.Errorf("flush Telegram image: %w", err)
+ }
case message.Text == "":
b.notice(telegramTextOnly)
case strings.TrimSpace(message.Text) == stopCommand:
return err
}
- err = l.Flush(ctx)
+ err = l.Flush(context.WithoutCancel(ctx))
if err != nil {
return fmt.Errorf("flush the message from Telegram: %w", err)
}
"io"
"net/http"
"net/url"
+ "path"
"strings"
"time"
)
return updates, err
}
+// GetImage downloads a file through the authenticated file endpoint. Errors
+// never include its token or URL, and the body is limited before storage.
+func (b *TelegramAPI) GetImage(ctx context.Context, id string) (data []byte, err error) {
+ var file struct {
+ Path string `json:"file_path"`
+ Size int64 `json:"file_size"`
+ }
+
+ err = b.call(ctx, b.sendTimeout, "getFile", map[string]any{"file_id": id}, &file)
+ if err != nil {
+ return nil, fmt.Errorf("get image file: %w", err)
+ }
+
+ if file.Size > imageMaxBytes {
+ return nil, errImageSize
+ }
+
+ if file.Path == "" || path.Clean(file.Path) != file.Path || strings.HasPrefix(file.Path, "/") || strings.HasPrefix(file.Path, "../") || strings.ContainsAny(file.Path, "?%#\\") {
+ return nil, errors.New("invalid image file path")
+ }
+
+ ctx, cancel := context.WithTimeout(ctx, b.sendTimeout)
+ defer cancel()
+
+ request, err := http.NewRequestWithContext(ctx, http.MethodGet, b.base+"/file/bot"+b.token+"/"+file.Path, nil)
+ if err != nil {
+ return nil, fmt.Errorf("make image request: %s", b.redact(err))
+ }
+
+ response, err := b.http.Do(request)
+ if err != nil {
+ return nil, b.networkError("image download failed", err, b.sendTimeout)
+ }
+
+ defer func() {
+ closeErr := response.Body.Close()
+ if closeErr != nil {
+ err = errors.Join(err, b.networkError("close image download", closeErr, b.sendTimeout))
+ }
+ }()
+
+ if response.StatusCode != http.StatusOK {
+ return nil, &TelegramError{
+ Status: response.StatusCode,
+ Description: http.StatusText(response.StatusCode),
+ }
+ }
+
+ data, err = io.ReadAll(io.LimitReader(response.Body, imageMaxBytes+1))
+ if err != nil {
+ return nil, b.networkError("image cut off", err, b.sendTimeout)
+ }
+
+ if len(data) > imageMaxBytes {
+ return nil, errImageSize
+ }
+
+ return data, nil
+}
+
// SendMessage sends text to chat with the format of entities, with no parse
// mode, so the text needs no escaping. With no entities, the text is plain. A
// call that takes longer than telegramSendTimeout fails with a
} `json:"parameters"`
}
-// TelegramMessage is a message to the bot. Text is empty for a message that is
-// not text, such as a photo.
+// TelegramMessage holds the text, caption, and image sizes of a bot message.
type TelegramMessage struct {
- Chat TelegramChat `json:"chat"`
- From *TelegramUser `json:"from"`
- Text string `json:"text"`
+ Chat TelegramChat `json:"chat"`
+ From *TelegramUser `json:"from"`
+ Text string `json:"text"`
+ Caption string `json:"caption"`
+ Photo []TelegramPhoto `json:"photo"`
+ Document *TelegramDocument `json:"document"`
+}
+
+// TelegramPhoto is one supplied size of a photo.
+type TelegramPhoto struct {
+ FileID string `json:"file_id"`
+ Width int `json:"width"`
+ Height int `json:"height"`
+ FileSize int64 `json:"file_size"`
+}
+
+// TelegramDocument is a file sent without photo compression.
+type TelegramDocument struct {
+ FileID string `json:"file_id"`
+ MimeType string `json:"mime_type"`
+ FileSize int64 `json:"file_size"`
}
// TelegramChat is the chat of a message. Type is private for a chat with one
stall func(call TelegramFakeCall) bool
added chan struct{}
stopped chan struct{}
+ images map[string][]byte
}
// Add makes update available to getUpdates.
return texts
}
+func (f *TelegramFake) Image(id string, data []byte) {
+ f.mutex.Lock()
+ defer f.mutex.Unlock()
+
+ if f.images == nil {
+ f.images = make(map[string][]byte)
+ }
+
+ f.images[id] = data
+}
+
func (f *TelegramFake) serve(w http.ResponseWriter, r *http.Request) {
+ id, file := strings.CutPrefix(r.URL.Path, "/file/bot"+telegramTestToken+"/")
+ if file {
+ f.mutex.Lock()
+
+ data, exists := f.images[id]
+ f.mutex.Unlock()
+
+ if !exists {
+ http.NotFound(w, r)
+
+ return
+ }
+
+ _, _ = w.Write(data)
+
+ return
+ }
+
method, found := strings.CutPrefix(r.URL.Path, "/bot"+telegramTestToken+"/")
if !found {
http.Error(w, `{"ok":false,"error_code":404,"description":"Not Found"}`, http.StatusNotFound)
call.ok = true
f.record(call)
+ if method == "getFile" {
+ _ = json.NewEncoder(w).Encode(map[string]any{
+ "ok": true,
+ "result": map[string]any{"file_path": params["file_id"]},
+ })
+
+ return
+ }
+
if method != "getUpdates" {
_, _ = io.WriteString(w, `{"ok":true,"result":true}`)
return
--- /dev/null
+package main
+
+import (
+ "context"
+ "errors"
+ "fmt"
+ "strings"
+)
+
+var (
+ errTelegramOtherMedia = errors.New("only text messages are supported")
+ errTelegramPhotoSize = errors.New("no photo size fits 1568 pixels")
+)
+
+// telegramImageText selects a supplied photo size, or an image document, and
+// stores it before returning marker text with the unchanged caption.
+func telegramImageText(ctx context.Context, api *TelegramAPI, store *ImageStore, message *TelegramMessage) (string, error) {
+ var (
+ id string
+ size int64
+ )
+
+ if len(message.Photo) > 0 {
+ var area int
+
+ for _, photo := range message.Photo {
+ if photo.Width <= 0 || photo.Height <= 0 || max(photo.Width, photo.Height) > 1568 {
+ continue
+ }
+
+ if photo.Width*photo.Height > area {
+ id = photo.FileID
+ size = photo.FileSize
+ area = photo.Width * photo.Height
+ }
+ }
+
+ if id == "" {
+ return "", errTelegramPhotoSize
+ }
+ } else if message.Document != nil {
+ document := message.Document
+ switch document.MimeType {
+ case "image/jpeg", "image/png", "image/webp":
+ id = document.FileID
+ size = document.FileSize
+ default:
+ if strings.HasPrefix(document.MimeType, "image/") {
+ return "", errImageFormat
+ }
+
+ return "", errTelegramOtherMedia
+ }
+ }
+
+ if size > imageMaxBytes {
+ return "", errImageSize
+ }
+
+ data, err := api.GetImage(ctx, id)
+ if err != nil {
+ return "", fmt.Errorf("download image: %w", err)
+ }
+
+ name, err := store.Save(data)
+ if err != nil {
+ return "", fmt.Errorf("store image: %w", err)
+ }
+
+ return imageText([]string{name}, message.Caption), nil
+}
--- /dev/null
+package main
+
+import (
+ "bytes"
+ "image"
+ "image/jpeg"
+ "net/http"
+ "os"
+ "path/filepath"
+ "strings"
+ "testing"
+ "time"
+
+ "github.com/stretchr/testify/require"
+)
+
+func TestTelegramSelectsLargestPhotoAtMost1568Pixels(t *testing.T) {
+ server := newModelFake(t)
+ test := newTestTurnLoop(t, server)
+ telegram := newTelegramFake(t)
+ bot, logs := telegramStartTestBot(t, telegram)
+ picture := imageTestPNG(t, 8, 6)
+ telegram.Image("selected", picture)
+
+ message := telegramOwnerMessage("")
+ message.Caption = "Read this label.\n"
+ message.Photo = []TelegramPhoto{
+ {
+ FileID: "small",
+ Width: 100,
+ Height: 200,
+ },
+ {
+ FileID: "too-large",
+ Width: 1600,
+ Height: 1200,
+ },
+ {
+ FileID: "selected",
+ Width: 1568,
+ Height: 1176,
+ },
+ {
+ FileID: "medium",
+ Width: 800,
+ Height: 600,
+ },
+ }
+ require.NoError(t, bot.handle(t.Context(), test.loop, nil, TelegramUpdate{
+ UpdateID: 1,
+ Message: message,
+ }))
+ test.events.WaitFor(t, "idle", 1)
+
+ files, err := os.ReadDir(test.agent.Images().dir)
+ require.NoError(t, err)
+ require.Len(t, files, 1)
+
+ logged, err := test.memory.Message(0)
+ require.NoError(t, err)
+ require.Equal(t, imageText([]string{files[0].Name()}, message.Caption), logged.Text)
+ require.True(t, strings.HasSuffix(logged.Text, "\n"))
+ require.Contains(t, string(server.Call(0).raw), "input_image")
+ require.NotContains(t, logs.String(), "base64")
+ require.NotContains(t, logs.String(), "Read this label")
+
+ for _, call := range telegram.Calls() {
+ if call.method == "getFile" {
+ require.Equal(t, "selected", call.params["file_id"])
+ }
+ }
+}
+
+func TestTelegramImageDocumentsUseContentFormatAndRejectOtherMedia(t *testing.T) {
+ telegram := newTelegramFake(t)
+ png := imageTestPNG(t, 2, 2)
+
+ var jpegBytes bytes.Buffer
+
+ require.NoError(t, jpeg.Encode(&jpegBytes, image.NewRGBA(image.Rect(0, 0, 2, 2)), nil))
+
+ // A made-up one-pixel lossless WebP image.
+ webp := []byte{'R', 'I', 'F', 'F', 22, 0, 0, 0, 'W', 'E', 'B', 'P', 'V', 'P', '8', 'L', 10, 0, 0, 0, 0x2f, 0, 0, 0, 0, 0x07, 0x10, 0x11, 0x11, 0x88}
+
+ telegram.Image("png", png)
+ telegram.Image("jpeg", jpegBytes.Bytes())
+ telegram.Image("webp", webp)
+ telegram.Image("invalid", []byte("not an image"))
+ telegram.Image("large", bytes.Repeat([]byte{0}, imageMaxBytes+1))
+
+ api := newTelegramAPI(telegram.URL, telegramTestToken)
+
+ store := &ImageStore{dir: filepath.Join(t.TempDir(), imageDirName)}
+ testCases := []struct {
+ name string
+ document TelegramDocument
+ suffix string
+ failure string
+ }{
+ {
+ name: "accepts PNG documents",
+ document: TelegramDocument{
+ FileID: "png",
+ MimeType: "image/png",
+ },
+ suffix: ".png]",
+ },
+ {
+ name: "accepts JPEG documents",
+ document: TelegramDocument{
+ FileID: "jpeg",
+ MimeType: "image/jpeg",
+ },
+ suffix: ".jpg]",
+ },
+ {
+ name: "accepts WebP documents",
+ document: TelegramDocument{
+ FileID: "webp",
+ MimeType: "image/webp",
+ },
+ suffix: ".webp]",
+ },
+ {
+ name: "other documents keep text answer",
+ document: TelegramDocument{
+ FileID: "pdf",
+ MimeType: "application/pdf",
+ },
+ failure: "only text messages are supported",
+ },
+ {
+ name: "other images refused",
+ document: TelegramDocument{
+ FileID: "gif",
+ MimeType: "image/gif",
+ },
+ failure: "JPEG, PNG, or WebP",
+ },
+ {
+ name: "invalid image refused",
+ document: TelegramDocument{
+ FileID: "invalid",
+ MimeType: "image/png",
+ },
+ failure: "JPEG, PNG, or WebP",
+ },
+ {
+ name: "large download refused",
+ document: TelegramDocument{
+ FileID: "large",
+ MimeType: "image/png",
+ },
+ failure: "exceeds 20 MB",
+ },
+ {
+ name: "large metadata refused",
+ document: TelegramDocument{
+ FileID: "not-downloaded",
+ MimeType: "image/png",
+ FileSize: imageMaxBytes + 1,
+ },
+ failure: "exceeds 20 MB",
+ },
+ }
+ for _, tc := range testCases {
+ t.Run(tc.name, func(t *testing.T) {
+ text, err := telegramImageText(t.Context(), api, store, &TelegramMessage{Document: &tc.document})
+ if tc.failure != "" {
+ require.ErrorContains(t, err, tc.failure)
+
+ return
+ }
+
+ require.NoError(t, err)
+ require.True(t, strings.HasSuffix(text, tc.suffix))
+ })
+ }
+}
+
+func TestTelegramRetriesImageDownloadBeforeAdvancingOffset(t *testing.T) {
+ telegram := newTelegramFake(t)
+ telegram.Image("photo", imageTestPNG(t, 2, 2))
+
+ tries := 0
+ telegram.fail = func(call TelegramFakeCall) (int, string) {
+ if call.method == "getFile" {
+ tries++
+ if tries == 1 {
+ return http.StatusServiceUnavailable, "try again"
+ }
+ }
+
+ return 0, ""
+ }
+ bot, logs := telegramStartTestBot(t, telegram)
+ bot.minWait = time.Millisecond
+ bot.maxWait = 5 * time.Millisecond
+ server := newModelFake(t)
+ socket := daemonStartTestWithBot(t, server, bot)
+ terminal := terminalStartTestClient(t, socket)
+ terminal.WaitFor(t, "[connected to OneChat]")
+
+ message := telegramOwnerMessage("")
+ message.Photo = []TelegramPhoto{{
+ FileID: "photo",
+ Width: 2,
+ Height: 2,
+ }}
+ telegram.Add(TelegramUpdate{
+ UpdateID: 7,
+ Message: message,
+ })
+ terminal.WaitFor(t, "[done]")
+ require.Eventually(t, func() bool {
+ offset, err := telegramLoadOffset(bot.offsetPath)
+ return err == nil && offset == 8
+ }, 10*time.Second, time.Millisecond)
+ require.Contains(t, logs.String(), "receive failed")
+ require.Contains(t, logs.String(), "receive back")
+ require.NotContains(t, logs.String(), telegramTestToken)
+ require.NotContains(t, logs.String(), "base64")
+}
+
+func TestTelegramRefusesPhotoWhenEverySizeIsTooLarge(t *testing.T) {
+ telegram := newTelegramFake(t)
+ _, err := telegramImageText(t.Context(), newTelegramAPI(telegram.URL, telegramTestToken), &ImageStore{dir: t.TempDir()}, &TelegramMessage{Photo: []TelegramPhoto{{
+ FileID: "too-big",
+ Width: 1600,
+ Height: 1000,
+ }}})
+ require.ErrorContains(t, err, "1568 pixels")
+ require.Empty(t, telegram.Calls())
+}
+
+func TestTelegramStorageFailureDoesNotAcceptTheImage(t *testing.T) {
+ telegram := newTelegramFake(t)
+ telegram.Image("photo", imageTestPNG(t, 2, 2))
+
+ test := newTestTurnLoop(t, newModelFake(t))
+ require.NoError(t, os.WriteFile(test.agent.Images().dir, []byte("blocked"), 0o600))
+
+ bot, _ := telegramStartTestBot(t, telegram)
+ message := telegramOwnerMessage("")
+ message.Photo = []TelegramPhoto{{
+ FileID: "photo",
+ Width: 2,
+ Height: 2,
+ }}
+ err := bot.handle(t.Context(), test.loop, nil, TelegramUpdate{
+ UpdateID: 1,
+ Message: message,
+ })
+ require.ErrorContains(t, err, "create image directory")
+ require.Equal(t, 0, test.memory.Len())
+
+ _, found := bot.next()
+ require.False(t, found)
+}
func newTestTurnLoopWith(t *testing.T, server *ModelFake, summarizer Summarizer) *TurnTestLoop {
t.Helper()
- agent, err := openAgent(t.TempDir())
+ agent, err := openAgent(filepath.Join(t.TempDir(), "agent"))
require.NoError(t, err)
agent.config.Workdir = t.TempDir()
projects / onechat / commitdiff