return &ImageStore{dir: filepath.Join(filepath.Dir(root), imageDirName)}
}
+// Voices returns the shared data-directory audio store, also for a child.
+func (a *Agent) Voices() *VoiceStore {
+ root := a.dir
+ if a.config.Parent != "" {
+ root = filepath.Dir(filepath.Dir(root))
+ }
+
+ return &VoiceStore{dir: filepath.Join(filepath.Dir(root), voiceDirName)}
+}
+
// CacheKey returns the prompt cache key of every model call of the agent.
func (a *Agent) CacheKey() string {
return "onechat-" + a.config.ID
err = errors.Join(err, agents.Close())
}()
+ whatsapp.inbox.voice = &VoiceTranscriber{config: config.Voice}
+
root := agents[0]
agent := root.agent
schedules := root.schedules
if hasBot {
telegram = newTelegramBot(newTelegramAPI(telegramAPIURL, botSettings.Token), botSettings.Owner, filepath.Join(agent.dir, telegramOffsetFileName), daemonLogger(out, "telegram: "))
+ telegram.voice = &VoiceTranscriber{config: config.Voice}
+
logger.Info("Telegram bot started")
}
func newTelegramBot(api *TelegramAPI, owner int64, offsetPath string, logger *slog.Logger) *TelegramBot {
return &TelegramBot{
api: api,
+ voice: &VoiceTranscriber{},
owner: owner,
offsetPath: offsetPath,
logger: logger,
// and the retries between them log nothing.
type TelegramBot struct {
api *TelegramAPI
+ voice *VoiceTranscriber
owner int64
offsetPath string
logger *slog.Logger
// /stop interrupts the turn, the line /stop NAME stops the subagent NAME, and a
// line that starts with /whatsapp runs a WhatsApp command on whatsapp, whose
// answers go to the chat only. Photos and image documents enter the shared
-// image store with their caption. Other media keep the text-only answer. Every
-// other update is ignored, and its content is not logged.
+// image store with their caption. Voice notes keep their audio and local
+// transcript. Other media keep the text-only answer. Every other update is
+// ignored, and its content is not logged.
func (b *TelegramBot) handle(ctx context.Context, l *TurnLoop, whatsapp *WhatsApp, update TelegramUpdate) error {
message := update.Message
if message == nil || message.Chat.Type != "private" || message.Chat.ID != b.owner || message.From == nil || message.From.ID != b.owner {
name, stop := stopSubagentName(message.Text)
args, isWhatsApp := whatsappCommandArgs(message.Text)
switch {
+ case message.Voice != nil:
+ text := telegramVoiceText(ctx, b.api, l.agent.Voices(), b.voice, message)
+ _, err := l.Submit(telegramSource, text)
+ if err != nil {
+ return fmt.Errorf("submit Telegram voice: %w", err)
+ }
+
+ err = l.Flush(context.WithoutCancel(ctx))
+ if err != nil {
+ return fmt.Errorf("flush Telegram voice: %w", err)
+ }
case len(message.Photo) > 0 || message.Document != nil:
text, err := telegramImageText(ctx, b.api, l.agent.Images(), message)
if err != nil {
// its own, because the long poll of getUpdates needs a longer limit.
telegramSendTimeout = 30 * time.Second
- // telegramFileLookupTimeout is the longest wait for image file details.
+ // telegramFileLookupTimeout is the longest wait for media file details.
telegramFileLookupTimeout = 30 * time.Second
- // telegramImageDownloadTimeout is the longest wait for image bytes.
- telegramImageDownloadTimeout = 30 * time.Second
+ // telegramFileDownloadTimeout is the longest wait for image bytes.
+ telegramFileDownloadTimeout = 30 * time.Second
// telegramTypingTimeout is the longest wait to show the typing action.
telegramTypingTimeout = 30 * time.Second
return updates, err
}
-// GetImage downloads a file through the authenticated file endpoint. Errors
-// never include its token or URL, and the body is limited before storage.
-func (b *TelegramAPI) GetImage(ctx context.Context, id string) (data []byte, err error) {
+// GetFile downloads bounded media through the authenticated file endpoint.
+// Errors never include its token or URL, and the body is limited before
+// storage.
+func (b *TelegramAPI) GetFile(ctx context.Context, id string, limit int64, tooLarge error) (data []byte, err error) {
var file struct {
Path string `json:"file_path"`
Size int64 `json:"file_size"`
err = b.call(ctx, telegramFileLookupTimeout, "getFile", map[string]any{"file_id": id}, &file)
if err != nil {
- return nil, fmt.Errorf("get image file: %w", err)
+ return nil, fmt.Errorf("get media file: %w", err)
}
- if file.Size > imageMaxBytes {
- return nil, errImageSize
+ if file.Size > limit {
+ return nil, tooLarge
}
if file.Path == "" || path.Clean(file.Path) != file.Path || strings.HasPrefix(file.Path, "/") || strings.HasPrefix(file.Path, "../") || strings.ContainsAny(file.Path, "?%#\\") {
- return nil, errors.New("invalid image file path")
+ return nil, errors.New("invalid media file path")
}
- ctx, cancel := context.WithTimeout(ctx, telegramImageDownloadTimeout)
+ ctx, cancel := context.WithTimeout(ctx, telegramFileDownloadTimeout)
defer cancel()
request, err := http.NewRequestWithContext(ctx, http.MethodGet, b.base+"/file/bot"+b.token+"/"+file.Path, nil)
if err != nil {
- return nil, fmt.Errorf("make image request: %s", b.redact(err))
+ return nil, fmt.Errorf("make media request: %s", b.redact(err))
}
response, err := b.http.Do(request)
if err != nil {
- return nil, b.networkError("image download failed", err, telegramImageDownloadTimeout)
+ return nil, b.networkError("media download failed", err, telegramFileDownloadTimeout)
}
defer func() {
closeErr := response.Body.Close()
if closeErr != nil {
- err = errors.Join(err, b.networkError("close image download", closeErr, telegramImageDownloadTimeout))
+ err = errors.Join(err, b.networkError("close media download", closeErr, telegramFileDownloadTimeout))
}
}()
}
}
- data, err = io.ReadAll(io.LimitReader(response.Body, imageMaxBytes+1))
+ data, err = io.ReadAll(io.LimitReader(response.Body, limit+1))
if err != nil {
- return nil, b.networkError("image cut off", err, telegramImageDownloadTimeout)
+ return nil, b.networkError("media cut off", err, telegramFileDownloadTimeout)
}
- if len(data) > imageMaxBytes {
- return nil, errImageSize
+ if int64(len(data)) > limit {
+ return nil, tooLarge
}
return data, nil
Caption string `json:"caption"`
Photo []TelegramPhoto `json:"photo"`
Document *TelegramDocument `json:"document"`
+ Voice *TelegramVoice `json:"voice"`
+}
+
+// TelegramVoice identifies a voice note in the authenticated file API.
+type TelegramVoice struct {
+ FileID string `json:"file_id"`
+ FileSize int64 `json:"file_size"`
}
// TelegramPhoto is one supplied size of a photo.
return "", errImageSize
}
- data, err := api.GetImage(ctx, id)
+ data, err := api.GetFile(ctx, id, imageMaxBytes, errImageSize)
if err != nil {
return "", fmt.Errorf("download image: %w", err)
}
--- /dev/null
+package main
+
+import "context"
+
+// telegramVoiceText keeps a voice note or a fixed failure marker. Download and
+// decode failures do not block later updates. Only owner messages reach it.
+func telegramVoiceText(ctx context.Context, api *TelegramAPI, store *VoiceStore, transcriber *VoiceTranscriber, message *TelegramMessage) string {
+ text := "(voice not downloaded)"
+ if message.Voice.FileSize <= voiceMaxBytes {
+ data, err := api.GetFile(ctx, message.Voice.FileID, voiceMaxBytes, errVoiceSize)
+ if err == nil {
+ text = transcriber.Text(ctx, store, data)
+ }
+ }
+
+ if message.Caption != "" {
+ text += "\n" + message.Caption
+ }
+
+ return text
+}
--- /dev/null
+package main
+
+import (
+ "os"
+ "path/filepath"
+ "testing"
+
+ "github.com/stretchr/testify/require"
+)
+
+func TestTelegramVoiceNotesReachTheAgentAsStoredText(t *testing.T) {
+ server := newModelFake(t)
+ test := newTestTurnLoop(t, server)
+ telegram := newTelegramFake(t)
+ bot, logs := telegramStartTestBot(t, telegram)
+ _, bot.voice = voiceTestInput(t, voiceTestScript)
+ audio := []byte("made-up Telegram voice")
+ telegram.Image("voice", audio)
+
+ message := telegramOwnerMessage("")
+ message.Voice = &TelegramVoice{FileID: "voice"}
+ message.Caption = "A voice caption."
+ require.NoError(t, bot.handle(t.Context(), test.loop, nil, TelegramUpdate{Message: message}))
+ test.events.WaitFor(t, "idle", 1)
+
+ logged, err := test.memory.Message(0)
+ require.NoError(t, err)
+ require.Contains(t, logged.Text, "[voice ")
+ require.Contains(t, logged.Text, "A made-up voice transcript.\nA voice caption.")
+
+ files, err := os.ReadDir(test.agent.Voices().dir)
+ require.NoError(t, err)
+ require.Len(t, files, 1)
+
+ saved, err := os.ReadFile(filepath.Join(test.agent.Voices().dir, files[0].Name()))
+ require.NoError(t, err)
+ require.Equal(t, audio, saved)
+ require.Contains(t, string(server.Call(0).raw), "A made-up voice transcript.")
+ require.NotContains(t, logs.String(), "A made-up")
+ require.NotContains(t, logs.String(), "A voice caption")
+}
+
+func TestTelegramVoiceFailuresKeepFollowingTextMoving(t *testing.T) {
+ testCases := []struct {
+ name string
+ size int64
+ file string
+ marker string
+ }{
+ {
+ name: "oversize metadata prevents a download",
+ size: voiceMaxBytes + 1,
+ file: "voice",
+ marker: "(voice not downloaded)",
+ },
+ {
+ name: "download failure becomes text",
+ file: "absent",
+ marker: "(voice not downloaded)",
+ },
+ {
+ name: "missing model preserves stored audio",
+ file: "voice",
+ marker: "transcriber or model is not available",
+ },
+ }
+ for _, tc := range testCases {
+ t.Run(tc.name, func(t *testing.T) {
+ test := newTestTurnLoop(t, newModelFake(t))
+ telegram := newTelegramFake(t)
+ bot, _ := telegramStartTestBot(t, telegram)
+ bot.voice.config.Model = filepath.Join(t.TempDir(), "missing")
+
+ telegram.Image("voice", []byte("made-up voice"))
+
+ message := telegramOwnerMessage("")
+ message.Voice = &TelegramVoice{
+ FileID: tc.file,
+ FileSize: tc.size,
+ }
+ require.NoError(t, bot.handle(t.Context(), test.loop, nil, TelegramUpdate{Message: message}))
+ require.NoError(t, bot.handle(t.Context(), test.loop, nil, TelegramUpdate{Message: telegramOwnerMessage("Next text.")}))
+ require.NoError(t, test.loop.Flush(t.Context()))
+
+ logged, err := test.memory.Message(0)
+ require.NoError(t, err)
+ require.Contains(t, logged.Text, tc.marker)
+
+ found := false
+ for n := 0; n < test.memory.Len(); n++ {
+ logged, err = test.memory.Message(n)
+ require.NoError(t, err)
+
+ found = found || logged.Text == "Next text."
+ }
+
+ require.True(t, found)
+ })
+ }
+}
+
+func TestTelegramIgnoresVoiceNotesFromOtherSenders(t *testing.T) {
+ test := newTestTurnLoop(t, newModelFake(t))
+ telegram := newTelegramFake(t)
+ bot, _ := telegramStartTestBot(t, telegram)
+ message := telegramOwnerMessage("")
+ message.Voice = &TelegramVoice{FileID: "voice"}
+ message.From.ID++
+ require.NoError(t, bot.handle(t.Context(), test.loop, nil, TelegramUpdate{Message: message}))
+ require.Empty(t, telegram.Calls())
+ require.Zero(t, test.memory.Len())
+}
"github.com/stretchr/testify/require"
)
+const voiceTestScript = `
+while [ "$1" != --output-file ]; do shift; done
+shift
+printf 'A made-up voice transcript.\n' > "$1.txt"
+`
+
func voiceTestInput(t *testing.T, script string) (*VoiceStore, *VoiceTranscriber) {
t.Helper()
require.Equal(t, "(voice not stored)", transcriber.Text(t.Context(), store, []byte("audio")))
}
+func TestVoiceBackupsKeepAudioButNotTemporaryTranscripts(t *testing.T) {
+ root := t.TempDir()
+ store := &VoiceStore{dir: filepath.Join(root, voiceDirName)}
+ name, err := store.Save([]byte("made-up backed-up audio"))
+ require.NoError(t, err)
+
+ temp := filepath.Join(store.dir, ".transcript-in-progress")
+ require.NoError(t, os.Mkdir(temp, 0o700))
+ require.NoError(t, os.WriteFile(filepath.Join(temp, "text.txt"), []byte("private temporary text"), 0o600))
+
+ target := t.TempDir()
+ require.NoError(t, backupCopy(t.Context(), root, target, filepath.Join(t.TempDir(), "backup.tar.gz")))
+ require.FileExists(t, filepath.Join(target, voiceDirName, name))
+ require.NoDirExists(t, filepath.Join(target, voiceDirName, ".transcript-in-progress"))
+
+ archive := filepath.Join(t.TempDir(), "voice.tar.gz")
+ _, err = backupArchive(t.Context(), target, archive)
+ require.NoError(t, err)
+
+ restored := t.TempDir()
+ require.NoError(t, backupExtract(t.Context(), archive, restored))
+
+ audio, err := os.ReadFile(filepath.Join(restored, voiceDirName, name))
+ require.NoError(t, err)
+ require.Equal(t, []byte("made-up backed-up audio"), audio)
+
+ info, err := os.Stat(filepath.Join(restored, voiceDirName, name))
+ require.NoError(t, err)
+ require.Equal(t, os.FileMode(0o600), info.Mode().Perm())
+}
+
+func TestVoiceConfigReadsExplicitLocalPaths(t *testing.T) {
+ dir := t.TempDir()
+ configWriteTest(t, dir, `{"voice":{"transcriber":"local-whisper","model":"model.bin"}}`)
+
+ config, err := loadConfig(dir)
+ require.NoError(t, err)
+ require.Equal(t, VoiceConfig{Transcriber: "local-whisper", Model: "model.bin"}, config.Voice)
+}
+
func TestVoiceLookupHonorsExplicitThenNixThenPath(t *testing.T) {
_, transcriber := voiceTestInput(t, "exit 0")
valid := transcriber.config
"context"
"fmt"
"log/slog"
+ "path/filepath"
"slices"
"strings"
"sync"
clock: clock,
groups: groups,
images: images,
+ voices: &VoiceStore{dir: filepath.Join(filepath.Dir(filepath.Dir(state.path)), voiceDirName)},
+ voice: &VoiceTranscriber{},
logger: logger,
wake: make(chan struct{}, 1),
failures: make(chan error, 1),
clock func() time.Time
groups *WhatsAppGroups
images *ImageStore
+ voices *VoiceStore
+ voice *VoiceTranscriber
logger *slog.Logger
wake chan struct{}
failures chan error
defer cancel()
message.Text = i.imageText(ctx, link, event, message.Text)
+ } else if event.Message.GetAudioMessage().GetPTT() {
+ message.Text = whatsappRedact(link.own, whatsappVoice(ctx, link.client, i.voices, i.voice, event.Message.GetAudioMessage()))
}
_, err := i.state.Add(chat, message, time.Now())
chatJID := info.Chat.ToNonAD()
switch {
case device == nil:
- case event.IsEdit, info.Edit != "", event.IsEphemeral, event.IsViewOnce, event.Message.GetImageMessage().GetViewOnce(), event.Message.GetVideoMessage().GetViewOnce(), event.SourceWebMsg != nil:
+ case event.IsEdit, info.Edit != "", event.IsEphemeral, event.IsViewOnce, event.Message.GetImageMessage().GetViewOnce(), event.Message.GetVideoMessage().GetViewOnce(), event.Message.GetAudioMessage().GetViewOnce(), event.SourceWebMsg != nil:
case chatJID.Server != types.DefaultUserServer && chatJID.Server != types.HiddenUserServer && chatJID.Server != types.GroupServer:
case slices.Contains(own, chatJID.User):
default:
contextInfo := extended.GetContextInfo()
switch {
+ case event.Message.GetAudioMessage().GetPTT():
+ text = "(voice not downloaded)"
+ contextInfo = event.Message.GetAudioMessage().GetContextInfo()
case event.Message.GetImageMessage() != nil:
image := event.Message.GetImageMessage()
text = image.GetCaption()
--- /dev/null
+package main
+
+import (
+ "context"
+ "time"
+
+ "go.mau.fi/whatsmeow/proto/waE2E"
+)
+
+// whatsappVoiceDownloadTimeout bounds one voice-note media download.
+const whatsappVoiceDownloadTimeout = 2 * time.Minute
+
+// whatsappVoice stores and transcribes accepted voice notes before their batch
+// is acknowledged. Failure markers reveal no URLs, keys, or process output.
+func whatsappVoice(ctx context.Context, client WhatsAppClient, store *VoiceStore, transcriber *VoiceTranscriber, audio *waE2E.AudioMessage) string {
+ if audio.GetFileLength() > voiceMaxBytes || client == nil {
+ return "(voice not downloaded)"
+ }
+
+ downloadCtx, cancel := context.WithTimeout(ctx, whatsappVoiceDownloadTimeout)
+ defer cancel()
+
+ data, err := client.Download(downloadCtx, audio)
+ if err != nil {
+ return "(voice not downloaded)"
+ }
+
+ return transcriber.Text(ctx, store, data)
+}
--- /dev/null
+package main
+
+import (
+ "errors"
+ "net/http"
+ "os"
+ "path/filepath"
+ "testing"
+ "testing/synctest"
+ "time"
+
+ "github.com/stretchr/testify/require"
+ "go.mau.fi/whatsmeow/proto/waE2E"
+ "go.mau.fi/whatsmeow/types"
+ "go.mau.fi/whatsmeow/types/events"
+)
+
+func TestWhatsAppVoiceNotesAreDurableBeforeAcknowledgement(t *testing.T) {
+ whatsapp, fake := whatsappStartTestLinked(t)
+ whatsapp.client = fake
+ _, whatsapp.inbox.voice = voiceTestInput(t, voiceTestScript)
+ audio := []byte("made-up WhatsApp voice")
+ fake.downloadData = audio
+ event := whatsappVoiceTestEvent(whatsappTestAna)
+ require.True(t, whatsapp.handle(event))
+ require.NoError(t, whatsapp.state.CloseAll())
+
+ restored := whatsappTestState(t, whatsapp.state.path)
+ require.Len(t, restored.Closed(), 1)
+
+ text := restored.Closed()[0].Messages[0].Text
+ require.Contains(t, text, "[voice ")
+ require.Contains(t, text, "A made-up voice transcript.")
+
+ files, err := os.ReadDir(whatsapp.inbox.voices.dir)
+ require.NoError(t, err)
+ require.Len(t, files, 1)
+
+ stored, err := os.ReadFile(filepath.Join(whatsapp.inbox.voices.dir, files[0].Name()))
+ require.NoError(t, err)
+ require.Equal(t, audio, stored)
+ require.True(t, whatsapp.handle(event))
+ require.NoError(t, whatsapp.state.CloseAll())
+ require.Len(t, whatsapp.state.Closed()[0].Messages, 1)
+}
+
+func TestWhatsAppVoiceRoutingSurvivesRestartWithoutDuplicateEntries(t *testing.T) {
+ dir := t.TempDir()
+ server := newModelFake(t)
+ server.reply = func(w http.ResponseWriter, _ ModelFakeCall) {
+ modelWriteEvents(w, modelCompletedEvent([]any{modelMessageItem("")}, 100, 0, 5))
+ }
+ first := newDaemonMessageTest(t, dir, server)
+ whatsappTestSession(t, dir, true)
+
+ whatsapp, fake, logs := whatsappStartTest(t, dir, newTestClock())
+ whatsapp.client = fake
+ _, whatsapp.inbox.voice = voiceTestInput(t, voiceTestScript)
+ fake.downloadData = []byte("made-up routed voice")
+ event := whatsappVoiceTestEvent(whatsappTestAna)
+ event.Info.Chat = whatsappTestFamily
+ event.Info.IsGroup = true
+
+ whatsapp.groups.set(whatsappTestFamily.String(), "Garden plans")
+ require.NoError(t, first.messages.routes.Set("whatsapp:"+whatsappTestFamily.String(), "Garden"))
+ require.True(t, whatsapp.handle(event))
+ require.NoError(t, whatsapp.state.CloseAll())
+
+ batch := whatsapp.state.Closed()[0]
+ text := batch.Text(time.UTC)
+ routed, err := first.messages.RouteWhatsApp(batch, text)
+ require.NoError(t, err)
+ require.True(t, routed)
+ first.Close(t)
+
+ second := newDaemonMessageTest(t, dir, server)
+ restored := whatsappRouteTestInbox(t, dir)
+ require.NoError(t, restored.deliver(t.Context(), second.loops[0], WhatsAppLink{}, true))
+ require.Empty(t, restored.state.Closed())
+ require.Zero(t, second.agents[0].memory.Len())
+ require.Equal(t, 1, second.agents[1].memory.Len())
+
+ message, err := second.agents[1].memory.Message(0)
+ require.NoError(t, err)
+ require.Equal(t, text, message.Text)
+ require.Contains(t, text, "[voice ")
+ require.Contains(t, text, "A made-up voice transcript.")
+ require.Equal(t, whatsapp.inbox.voices.dir, second.agents[1].agent.Voices().dir)
+ second.Start(t, 1)
+ second.events[1].WaitFor(t, "idle", 1)
+ require.Contains(t, string(server.Call(0).raw), "A made-up voice transcript.")
+ require.NotContains(t, logs.String()+second.logs.String(), "A made-up voice transcript")
+ second.Close(t)
+
+ third := newDaemonMessageTest(t, dir, server)
+ require.Equal(t, 1, third.agents[1].memory.Len())
+ require.False(t, third.loops[1].waiting())
+}
+
+func TestWhatsAppVoiceFailuresKeepLaterMessagesMoving(t *testing.T) {
+ synctest.Test(t, func(t *testing.T) {
+ whatsapp, fake := whatsappStartTestLinked(t)
+ whatsapp.client = fake
+ fake.downloadWait = 3 * time.Minute
+
+ require.True(t, whatsapp.handle(whatsappVoiceTestEvent(whatsappTestAna)))
+
+ fake.downloadWait = 0
+ fake.downloadErr = errors.New("secret URL and key")
+ event := whatsappVoiceTestEvent(whatsappTestAna)
+ event.Info.ID = "failed"
+ require.True(t, whatsapp.handle(event))
+ require.True(t, whatsapp.handle(whatsappTestMessage(whatsappTestAna, whatsappTestAna, "next", "Next text.", event.Info.Timestamp)))
+ require.NoError(t, whatsapp.state.CloseAll())
+
+ messages := whatsapp.state.Closed()[0].Messages
+ require.Len(t, messages, 3)
+ require.Equal(t, "(voice not downloaded)", messages[0].Text)
+ require.Equal(t, "(voice not downloaded)", messages[1].Text)
+ require.Equal(t, "Next text.", messages[2].Text)
+ })
+}
+
+func TestWhatsAppSkipsViewOnceAndNonVoiceAudio(t *testing.T) {
+ whatsapp, fake := whatsappStartTestLinked(t)
+ whatsapp.client = fake
+ event := whatsappVoiceTestEvent(whatsappTestAna)
+ event.Message.AudioMessage.ViewOnce = new(true)
+ require.True(t, whatsapp.handle(event))
+
+ event.Message.AudioMessage.ViewOnce = new(false)
+ event.Message.AudioMessage.PTT = new(false)
+ require.True(t, whatsapp.handle(event))
+ require.Empty(t, fake.Downloads())
+ require.NoError(t, whatsapp.state.CloseAll())
+ require.Empty(t, whatsapp.state.Closed())
+}
+
+func whatsappVoiceTestEvent(sender types.JID) *events.Message {
+ event := whatsappTestMessage(whatsappTestAna, sender, "voice", "", time.Date(2026, 1, 1, 12, 0, 0, 0, time.UTC))
+ event.Message = &waE2E.Message{AudioMessage: &waE2E.AudioMessage{
+ PTT: new(true),
+ Mimetype: new("audio/ogg; codecs=opus"),
+ }}
+
+ return event
+}
projects / onechat / commitdiff