Send the replies to Telegram with their formats

The bot converts the Markdown of each model reply to text and message
entities, and sends them with the entities field of sendMessage, with no
parse mode. The phone then shows bold text, code and code blocks
formatted, not their marks. A reply longer than 4096 UTF-16 units is
split as before, and an entity that crosses a cut becomes one entity in
each part, with offsets from the start of its part.

When Telegram refuses a part with entities with HTTP 400, the bot sends
the same text again once with no entities and logs one line without the
text, so a bad entity does not lose a reply. Other failures wait and try
again as before.

Only the replies get entities. The notices of the bot get their own
event kind, so they stay plain text with the errors and the messages
typed in the terminal. A reply with no text left after the conversion
goes as its Markdown. The log and the terminal client do not change.

Issue: 3309f9b

diff --git a/README.rst b/README.rst
index 4bda34e7a09b06332c4726c6f047f31d7ce16cd4..c9aaaeeb4e1b59fa0e4ac307118508f7d7c4010f 100644 (file)
--- a/README.rst
+++ b/README.rst
@@ -131,11 +131,14 @@ new messages, so it needs no open port.
 
 The bot reads only the private chat of the owner and ignores all other
 updates. Each text message goes to the agent as a user message, and the
 
 The bot reads only the private chat of the owner and ignores all other
 updates. Each text message goes to the agent as a user message, and the

-line ``/stop`` interrupts the running turn. The bot sends the replies as
-plain text, in parts of at most 4096 characters, and each error as a
-short message. It shows the typing action while a turn runs. It does not
-send the tool calls. It sends each message that the user types in the
-terminal client, with the mark ``[terminal]``.
+line ``/stop`` interrupts the running turn. The bot sends the replies in
+parts of at most 4096 characters, with their Markdown as Telegram
+formats: bold, italic, strikethrough, code, code blocks, links, and
+headings as bold text. When Telegram refuses the formats of a part, the
+bot sends the part again as plain text. Each error goes as a short
+message in plain text. It shows the typing action while a turn runs. It
+does not send the tool calls. It sends each message that the user types
+in the terminal client, with the mark ``[terminal]``.

 
 ``agent/telegram-offset.json`` holds the ID of the next update. The bot
 moves it past an update only after the message is durable in the log, so
 
 ``agent/telegram-offset.json`` holds the ID of the next update. The bot
 moves it past an update only after the message is durable in the log, so

diff --git a/botapi.go b/botapi.go
index 57bf9dba8111e163f99bda5a78470dafd594016e..f48465c3174674173cb1b8d8337932cdae5c05ba 100644 (file)
--- a/botapi.go
+++ b/botapi.go
@@ -130,16 +130,23 @@ func (b *botAPI) GetUpdates(ctx context.Context, offset int64) ([]botUpdate, err
        return updates, err
 }
 
        return updates, err
 }
 

-// SendMessage sends text to chat as plain text, with no parse mode. A call that
-// takes longer than sendTimeout fails with errBotNetwork.
-func (b *botAPI) SendMessage(ctx context.Context, chat int64, text string) error {
+// SendMessage sends text to chat with the format of entities, with no parse
+// mode, so the text needs no escaping. With no entities, the text is plain. A
+// call that takes longer than sendTimeout fails with errBotNetwork.
+func (b *botAPI) SendMessage(ctx context.Context, chat int64, text string, entities []botEntity) error {

        ctx, cancel := context.WithTimeout(ctx, b.sendTimeout)
        defer cancel()
 
        ctx, cancel := context.WithTimeout(ctx, b.sendTimeout)
        defer cancel()
 

-       return b.call(ctx, "sendMessage", map[string]any{
+       params := map[string]any{

                "chat_id": chat,
                "text":    text,
                "chat_id": chat,
                "text":    text,

-       }, nil)
+       }
+
+       if len(entities) > 0 {
+               params["entities"] = entities
+       }
+
+       return b.call(ctx, "sendMessage", params, nil)

 }
 
 // SendTyping shows the typing action in chat for about five seconds. A call
 }
 
 // SendTyping shows the typing action in chat for about five seconds. A call

diff --git a/botapi_test.go b/botapi_test.go
index e1827471b425478352ce0200897b84f22e026dc7..9c4db5461c2d1434be43d0577ff0753b012f498b 100644 (file)
--- a/botapi_test.go
+++ b/botapi_test.go
@@ -48,7 +48,7 @@ func TestGetUpdatesAsksForMessagesFromTheOffset(t *testing.T) {
 
 func TestSendMessageSendsPlainText(t *testing.T) {
        server := newFakeBot(t)
 
 func TestSendMessageSendsPlainText(t *testing.T) {
        server := newFakeBot(t)

-       err := newBotAPI(server.URL, testBotToken).SendMessage(t.Context(), testOwner, "*not bold*")
+       err := newBotAPI(server.URL, testBotToken).SendMessage(t.Context(), testOwner, "*not bold*", nil)

        require.NoError(t, err)
 
        require.Equal(t, map[string]any{
        require.NoError(t, err)
 
        require.Equal(t, map[string]any{


@@ -57,6 +57,28 @@ func TestSendMessageSendsPlainText(t *testing.T) {
        }, server.Calls()[0].params)
 }
 
        }, server.Calls()[0].params)
 }
 

+func TestSendMessageSendsTheEntities(t *testing.T) {
+       server := newFakeBot(t)
+       err := newBotAPI(server.URL, testBotToken).SendMessage(t.Context(), testOwner, "a link", []botEntity{
+               {
+                       Type:   "text_link",
+                       Offset: 2,
+                       Length: 4,
+                       URL:    "https://example.com",
+               },
+       })
+       require.NoError(t, err)
+
+       require.Equal(t, []any{
+               map[string]any{
+                       "type":   "text_link",
+                       "offset": float64(2),
+                       "length": float64(4),
+                       "url":    "https://example.com",
+               },
+       }, server.Calls()[0].params["entities"])
+}
+

 func TestSendMessageTimesOutOnAStalledConnection(t *testing.T) {
        server := newFakeBot(t)
        server.stall = func(call fakeBotCall) bool {
 func TestSendMessageTimesOutOnAStalledConnection(t *testing.T) {
        server := newFakeBot(t)
        server.stall = func(call fakeBotCall) bool {


@@ -66,7 +88,7 @@ func TestSendMessageTimesOutOnAStalledConnection(t *testing.T) {
        api := newBotAPI(server.URL, testBotToken)
        api.sendTimeout = 50 * time.Millisecond
        start := time.Now()
        api := newBotAPI(server.URL, testBotToken)
        api.sendTimeout = 50 * time.Millisecond
        start := time.Now()

-       err := api.SendMessage(t.Context(), testOwner, "hi")
+       err := api.SendMessage(t.Context(), testOwner, "hi", nil)

        require.ErrorIs(t, err, errBotNetwork)
        require.Less(t, time.Since(start), 5*time.Second)
        require.NotContains(t, err.Error(), testBotToken)
        require.ErrorIs(t, err, errBotNetwork)
        require.Less(t, time.Since(start), 5*time.Second)
        require.NotContains(t, err.Error(), testBotToken)


@@ -79,7 +101,7 @@ func TestSendMessageTimesOutOnAStalledConnection(t *testing.T) {
        ctx, cancel := context.WithCancel(t.Context())
        time.AfterFunc(50*time.Millisecond, cancel)
        start = time.Now()
        ctx, cancel := context.WithCancel(t.Context())
        time.AfterFunc(50*time.Millisecond, cancel)
        start = time.Now()

-       err = api.SendMessage(ctx, testOwner, "hi")
+       err = api.SendMessage(ctx, testOwner, "hi", nil)

        require.Error(t, err)
        require.Less(t, time.Since(start), 5*time.Second)
 }
        require.Error(t, err)
        require.Less(t, time.Since(start), 5*time.Second)
 }


@@ -148,7 +170,7 @@ func TestBotErrorsDoNotHoldTheToken(t *testing.T) {
                        }))
                        t.Cleanup(server.Close)
 
                        }))
                        t.Cleanup(server.Close)
 

-                       err := newBotAPI(server.URL, testBotToken).SendMessage(t.Context(), testOwner, "hi")
+                       err := newBotAPI(server.URL, testBotToken).SendMessage(t.Context(), testOwner, "hi", nil)

                        require.ErrorContains(t, err, tc.errorText)
                        require.NotContains(t, err.Error(), testBotToken)
                        require.NotContains(t, err.Error(), server.URL)
                        require.ErrorContains(t, err, tc.errorText)
                        require.NotContains(t, err.Error(), testBotToken)
                        require.NotContains(t, err.Error(), server.URL)

diff --git a/telegram.go b/telegram.go
index dabf2e8a70be50b919fcd9006522e2cf8d6a709b..9a24a4e861d7436bcd0a109480b50fbbb538377e 100644 (file)
--- a/telegram.go
+++ b/telegram.go
@@ -8,6 +8,7 @@ import (
        "fmt"
        "io"
        "log"
        "fmt"
        "io"
        "log"

+       "net/http"

        "os"
        "path/filepath"
        "strings"
        "os"
        "path/filepath"
        "strings"


@@ -50,9 +51,10 @@ type botConfig struct {
 // telegramBot connects the private chat of the owner with the agent. It polls
 // the Bot API for new messages and sends each text message of the owner to the
 // agent. It ignores every other update. Its sender sends the replies, the
 // telegramBot connects the private chat of the owner with the agent. It polls
 // the Bot API for new messages and sends each text message of the owner to the
 // agent. It ignores every other update. Its sender sends the replies, the

-// errors, and the user messages of the other clients to the chat as plain text,
-// and shows the typing action while a turn runs. Tool calls do not go to the
-// chat.
+// errors, the notices of the bot, and the user messages of the other clients to
+// the chat, and shows the typing action while a turn runs. Only the replies
+// have a format: their Markdown goes as message entities. The rest is plain
+// text. Tool calls do not go to the chat.

 //
 // The offset file holds the ID of the next update. It moves past an update
 // only after the message of the update is durable in the log, so a restart
 //
 // The offset file holds the ID of the next update. It moves past an update
 // only after the message of the update is durable in the log, so a restart


@@ -74,6 +76,13 @@ type telegramBot struct {
        ready chan struct{}
 }
 
        ready chan struct{}
 }
 

+// botPart is the text of one message to the chat, at most messageUnits long,
+// and its entities.
+type botPart struct {
+       text     string
+       entities []botEntity
+}
+

 // offsetFile is the content of the offset file.
 type offsetFile struct {
        Offset int64 `json:"offset"`
 // offsetFile is the content of the offset file.
 type offsetFile struct {
        Offset int64 `json:"offset"`


@@ -156,7 +165,7 @@ func newTelegramBot(api *botAPI, owner int64, offsetPath string, logger *log.Log
 // stop the agent.
 func (b *telegramBot) Notify(event Event) {
        switch event.Kind {
 // stop the agent.
 func (b *telegramBot) Notify(event Event) {
        switch event.Kind {

-       case "busy", "idle", "reply", "error":
+       case "busy", "idle", "reply", "error", "notice":

        case "user":
                if event.Source == telegramSource {
                        return
        case "user":
                if event.Source == telegramSource {
                        return


@@ -228,7 +237,7 @@ func (b *telegramBot) send(ctx context.Context) {
                case "idle":
                        busy = false
                default:
                case "idle":
                        busy = false
                default:

-                       for _, part := range splitMessage(botText(event), messageUnits) {
+                       for _, part := range botParts(event) {

                                b.deliver(ctx, part)
                        }
 
                                b.deliver(ctx, part)
                        }
 


@@ -260,16 +269,26 @@ func (b *telegramBot) typing(ctx context.Context) {
        _ = b.api.SendTyping(ctx, b.owner)
 }
 
        _ = b.api.SendTyping(ctx, b.owner)
 }
 

-// deliver sends text to the chat. It tries again after each failure that a
-// later try can fix, until ctx ends. It drops the text after any other failure.
-func (b *telegramBot) deliver(ctx context.Context, text string) {
+// deliver sends part to the chat. When Telegram refuses a part with entities
+// (HTTP 400), it sends the text of the part again once with no entities, so a
+// reply is not lost to a bad entity. It tries again after each failure that a
+// later try can fix, until ctx ends. It drops the part after any other failure.
+func (b *telegramBot) deliver(ctx context.Context, part botPart) {

        wait := b.minWait
        for {
        wait := b.minWait
        for {

-               err := b.api.SendMessage(ctx, b.owner, text)
+               err := b.api.SendMessage(ctx, b.owner, part.text, part.entities)

                if err == nil || ctx.Err() != nil {
                        return
                }
 
                if err == nil || ctx.Err() != nil {
                        return
                }
 

+               var answer *botError
+               if len(part.entities) > 0 && errors.As(err, &answer) && answer.Status == http.StatusBadRequest {
+                       b.logger.Printf("entities refused, the message goes again as plain text: %v", err)
+                       part.entities = nil
+
+                       continue
+               }
+

                retry, asked := retryable(err)
                if !retry {
                        b.logger.Printf("message dropped: %v", err)
                retry, asked := retryable(err)
                if !retry {
                        b.logger.Printf("message dropped: %v", err)


@@ -379,7 +398,7 @@ func (b *telegramBot) handle(ctx context.Context, l *loop, update botUpdate) err
 // notice queues a short answer of the bot for the chat.
 func (b *telegramBot) notice(text string) {
        b.Notify(Event{
 // notice queues a short answer of the bot for the chat.
 func (b *telegramBot) notice(text string) {
        b.Notify(Event{

-               Kind: "reply",
+               Kind: "notice",

                Text: text,
        })
 }
                Text: text,
        })
 }


@@ -422,6 +441,23 @@ func saveOffset(path string, offset int64) error {
        return nil
 }
 
        return nil
 }
 

+// botParts returns the messages of event for the chat. The Markdown of a reply
+// becomes text with entities. Every other event is plain text. A reply that has
+// no text left after the conversion, such as an empty code block, goes as its
+// Markdown.
+func botParts(event Event) []botPart {
+       text, entities := botText(event), []botEntity(nil)
+       if event.Kind == "reply" {
+               text, entities = markdownEntities(event.Text)
+       }
+
+       if strings.TrimSpace(text) == "" {
+               text, entities = event.Text, nil
+       }
+
+       return splitEntities(text, entities, messageUnits)
+}
+

 // botText returns the text of event for the chat. A user message of another
 // client has its source as a mark.
 func botText(event Event) string {
 // botText returns the text of event for the chat. A user message of another
 // client has its source as a mark.
 func botText(event Event) string {


@@ -435,6 +471,38 @@ func botText(event Event) string {
        }
 }
 
        }
 }
 

+// splitEntities cuts text into parts with splitMessage, and gives each part the
+// entities in it, with offsets from the start of the part. An entity that
+// crosses a cut becomes one entity in each part.
+func splitEntities(text string, entities []botEntity, limit int) []botPart {
+       var parts []botPart
+       start := 0
+       for _, part := range splitMessage(text, limit) {
+               end := start + utf16Units(part)
+               var kept []botEntity
+               for _, entity := range entities {
+                       from := max(entity.Offset, start)
+                       to := min(entity.Offset+entity.Length, end)
+                       if to <= from {
+                               continue
+                       }
+
+                       entity.Offset = from - start
+                       entity.Length = to - from
+                       kept = append(kept, entity)
+               }
+
+               parts = append(parts, botPart{
+                       text:     part,
+                       entities: kept,
+               })
+
+               start = end
+       }
+
+       return parts
+}
+

 // splitMessage cuts text into parts of at most limit UTF-16 code units, the
 // unit of the Telegram message limit. It cuts after the last line break or
 // space of a part when one is in its second half, and else at the limit. It
 // splitMessage cuts text into parts of at most limit UTF-16 code units, the
 // unit of the Telegram message limit. It cuts after the last line break or
 // space of a part when one is in its second half, and else at the limit. It

diff --git a/telegram_test.go b/telegram_test.go
index b1cd6ae7348b9c9aedccb77678330fa55a12d54a..2578b77b1c97b98561f3718e56a6c041ec2cc6a7 100644 (file)
--- a/telegram_test.go
+++ b/telegram_test.go
@@ -217,7 +217,9 @@ func TestDeliverTriesAgainAfterATimeout(t *testing.T) {
 
        bot, logs := startTestBot(t, telegram)
        bot.api.sendTimeout = 50 * time.Millisecond
 
        bot, logs := startTestBot(t, telegram)
        bot.api.sendTimeout = 50 * time.Millisecond

-       bot.deliver(t.Context(), "hello")
+       bot.deliver(t.Context(), botPart{
+               text: "hello",
+       })

 
        require.Equal(t, []string{"hello"}, telegram.Sent())
        require.Len(t, telegram.Calls(), 2)
 
        require.Equal(t, []string{"hello"}, telegram.Sent())
        require.Len(t, telegram.Calls(), 2)


@@ -225,6 +227,81 @@ func TestDeliverTriesAgainAfterATimeout(t *testing.T) {
        require.NotContains(t, logs.String(), testBotToken)
 }
 
        require.NotContains(t, logs.String(), testBotToken)
 }
 

+func TestOnlyRepliesGetEntities(t *testing.T) {
+       server := newFakeResponses(t)
+       server.reply = func(w http.ResponseWriter, call fakeCall) {
+               writeEvents(w, completedEvent([]any{messageItem("**Done** with `x`.")}, 100, 0, 5))
+       }
+
+       telegram := newFakeBot(t)
+       bot, _ := startTestBot(t, telegram)
+       socket := startTestDaemonWithBot(t, server, bot)
+       terminal := startTestClient(t, socket)
+       terminal.WaitFor(t, "[connected to OneChat]")
+
+       terminal.Type("make **it** so")
+       waitForSent(t, telegram, "[terminal] make **it** so", "Done with x.")
+
+       telegram.Add(botUpdate{
+               UpdateID: 1,
+               Message:  ownerMessage(""),
+       })
+
+       waitForSent(t, telegram, textOnly)
+
+       entities := map[string]any{}
+       for _, call := range telegram.Calls() {
+               if call.method == "sendMessage" {
+                       entities[call.params["text"].(string)] = call.params["entities"]
+               }
+       }
+
+       require.Equal(t, map[string]any{
+               "[terminal] make **it** so": nil,
+               "Done with x.": []any{
+                       map[string]any{
+                               "type":   "bold",
+                               "offset": float64(0),
+                               "length": float64(4),
+                       },
+                       map[string]any{
+                               "type":   "code",
+                               "offset": float64(10),
+                               "length": float64(1),
+                       },
+               },
+               textOnly: nil,
+       }, entities)
+}
+
+func TestRefusedEntitiesGoAgainAsPlainText(t *testing.T) {
+       telegram := newFakeBot(t)
+       telegram.fail = func(call fakeBotCall) (int, string) {
+               if call.params["entities"] == nil {
+                       return 0, ""
+               }
+
+               return http.StatusBadRequest, "Bad Request: can't parse entities of bot" + testBotToken
+       }
+
+       bot, logs := startTestBot(t, telegram)
+       text, entities := markdownEntities("a **secret** word")
+       bot.deliver(t.Context(), botPart{
+               text:     text,
+               entities: entities,
+       })
+
+       calls := telegram.Calls()
+       require.Len(t, calls, 2)
+       require.NotNil(t, calls[0].params["entities"])
+       require.Nil(t, calls[1].params["entities"])
+       require.Equal(t, []string{"a secret word"}, telegram.Sent())
+       require.Equal(t, 1, strings.Count(logs.String(), "\n"))
+       require.Contains(t, logs.String(), "entities refused")
+       require.NotContains(t, logs.String(), testBotToken)
+       require.NotContains(t, logs.String(), "secret word")
+}
+

 func TestBotLogsNoTokenWhenTheNetworkFails(t *testing.T) {
        telegram := newFakeBot(t)
        telegram.Close()
 func TestBotLogsNoTokenWhenTheNetworkFails(t *testing.T) {
        telegram := newFakeBot(t)
        telegram.Close()


@@ -289,6 +366,49 @@ func TestSplitMessage(t *testing.T) {
        require.Equal(t, long, strings.Join(parts, ""))
 }
 
        require.Equal(t, long, strings.Join(parts, ""))
 }
 

+func TestSplitEntitiesCutsAnEntityAcrossASplit(t *testing.T) {
+       parts := splitEntities("aabbbbcc", []botEntity{
+               {
+                       Type:   "bold",
+                       Offset: 2,
+                       Length: 4,
+               },
+               {
+                       Type:   "italic",
+                       Offset: 6,
+                       Length: 2,
+               },
+       }, 5)
+
+       require.Equal(t, []botPart{
+               {
+                       text: "aabbb",
+                       entities: []botEntity{
+                               {
+                                       Type:   "bold",
+                                       Offset: 2,
+                                       Length: 3,
+                               },
+                       },
+               },
+               {
+                       text: "bcc",
+                       entities: []botEntity{
+                               {
+                                       Type:   "bold",
+                                       Offset: 0,
+                                       Length: 1,
+                               },
+                               {
+                                       Type:   "italic",
+                                       Offset: 1,
+                                       Length: 2,
+                               },
+                       },
+               },
+       }, parts)
+}
+

 func TestLoadBotConfig(t *testing.T) {
        testCases := []struct {
                name      string
 func TestLoadBotConfig(t *testing.T) {
        testCases := []struct {
                name      string