Escape tags in rich replies as HTML entities

The bot writes each < of a reply outside inline code and fenced code
blocks as &lt; and each & as &amp; before it sends the reply with
sendRichMessage. Rich Markdown does not read a backslash before < as an
escape: it shows the backslash and still reads the tag as HTML. It reads
HTML entities, so &lt; shows as <. The escape of & makes an entity that
the model writes in prose, such as &lt;, show as written.

The entity fallback reads no HTML, so a rich part that Telegram refuses
goes to the fallback with the escape reversed, and the user sees the
text as the model wrote it. The reverse is exact, because after the
escape each & outside code starts an entity of the escape. The escape
and its reverse find the code of a text in the same way, so the reverse
also works on each part of the split.

The split still counts the escaped text against the rich limit, and the
fallback splits the text without the escape at its own limit. When the
split cuts a line longer than the limit, it cuts before an & in the last
four bytes, so that it does not cut an entity.

Issue: d75e312

diff --git a/telegram.go b/telegram.go
index 13c200c8481a1c47c6cb1497a7e9f84a4a869a62..6d5332de8c9868eeabc0210a566063be8ae320f7 100644 (file)
--- a/telegram.go
+++ b/telegram.go
@@ -311,9 +311,9 @@ func (b *TelegramBot) typing(ctx context.Context) {
 }
 
 // post sends event to the chat. A reply goes as rich messages of its Markdown,
 }
 
 // post sends event to the chat. A reply goes as rich messages of its Markdown,

-// with each < outside code escaped by telegramEscapeTags and then split with
-// telegramSplitRich, so that the length limit counts the escaped text. Every
-// other event goes as plain text, split with telegramSplitMessage.
+// with each < and & outside code escaped by telegramEscapeTags and then split
+// with telegramSplitRich, so that the length limit counts the escaped text.
+// Every other event goes as plain text, split with telegramSplitMessage.

 func (b *TelegramBot) post(ctx context.Context, event Event) {
        if event.Kind != "reply" {
                for _, text := range telegramSplitMessage(telegramText(event), telegramMessageUnits) {
 func (b *TelegramBot) post(ctx context.Context, event Event) {
        if event.Kind != "reply" {
                for _, text := range telegramSplitMessage(telegramText(event), telegramMessageUnits) {


@@ -411,11 +411,13 @@ func (b *TelegramBot) deliver(ctx context.Context, part TelegramPart) {
        }
 }
 
        }
 }
 

-// deliverRich sends markdown to the chat as a rich message. When the Bot API
-// refuses it with HTTP 400 or 404, the message was not sent, so deliverRich
-// sends the same Markdown once with message entities, and the reply is not
-// lost and does not repeat. It tries again after each failure that a later try
-// can fix, until ctx ends. It drops the message after any other failure.
+// deliverRich sends markdown, escaped with telegramEscapeTags, to the chat as a
+// rich message. When the Bot API refuses it with HTTP 400 or 404, the message
+// was not sent, so deliverRich sends the text of the part once with message
+// entities, and the reply is not lost and does not repeat. Message entities
+// read no HTML, so the fallback gets the Markdown of telegramUnescapeTags, as
+// the model wrote it. It tries again after each failure that a later try can
+// fix, until ctx ends. It drops the message after any other failure.

 func (b *TelegramBot) deliverRich(ctx context.Context, markdown string) {
        wait := b.minWait
        for {
 func (b *TelegramBot) deliverRich(ctx context.Context, markdown string) {
        wait := b.minWait
        for {


@@ -426,7 +428,7 @@ func (b *TelegramBot) deliverRich(ctx context.Context, markdown string) {
 
                if telegramRejected(err) {
                        b.logger.Printf("rich message refused, the reply goes with entities: %v", err)
 
                if telegramRejected(err) {
                        b.logger.Printf("rich message refused, the reply goes with entities: %v", err)

-                       for _, part := range telegramReplyParts(markdown) {
+                       for _, part := range telegramReplyParts(telegramUnescapeTags(markdown)) {

                                b.deliver(ctx, part)
                        }
 
                                b.deliver(ctx, part)
                        }
 


@@ -546,7 +548,8 @@ func telegramText(event Event) string {
 // after the last line break outside a code block, then after the last line
 // break in a code block, which it closes at the end of the part and opens again
 // in the next part with the same fence line. A line longer than limit is cut at
 // after the last line break outside a code block, then after the last line
 // break in a code block, which it closes at the end of the part and opens again
 // in the next part with the same fence line. A line longer than limit is cut at

-// the limit. It never cuts a character.
+// the limit, or before an & in the four bytes before the limit, so that it does
+// not cut an entity of telegramEscapeTags. It never cuts a character.

 func telegramSplitRich(markdown string, limit int) []string {
        var parts []string
        for len(markdown) > limit {
 func telegramSplitRich(markdown string, limit int) []string {
        var parts []string
        for len(markdown) > limit {


@@ -649,6 +652,12 @@ func telegramRichCut(markdown string, limit int) (int, string, string) {
                cut--
        }
 
                cut--
        }
 

+       start := max(cut-len("&amp;")+1, 0)
+       entity := strings.LastIndexByte(markdown[start:cut], '&')
+       if entity >= 0 && start+entity > 0 {
+               cut = start + entity
+       }
+

        return cut, "", ""
 }
 
        return cut, "", ""
 }
 

diff --git a/telegram_markdown.go b/telegram_markdown.go
index 16b83f2f851d81bddd19353a1af8e6c43dfb6c97..8738bcdb38f57e673bd195e11d630505bbee0074 100644 (file)
--- a/telegram_markdown.go
+++ b/telegram_markdown.go
@@ -11,6 +11,15 @@ import (
 // plain.
 const telegramMarkdownPunctuation = "!\"#$%&'()*+,-./:;<=>?@[\\]^_`{|}~"
 
 // plain.
 const telegramMarkdownPunctuation = "!\"#$%&'()*+,-./:;<=>?@[\\]^_`{|}~"
 

+var (
+       // telegramTagEscaper writes the characters that rich Markdown reads as HTML
+       // as HTML entities.
+       telegramTagEscaper = strings.NewReplacer("&", "&amp;", "<", "&lt;")
+
+       // telegramTagUnescaper reverses telegramTagEscaper.
+       telegramTagUnescaper = strings.NewReplacer("&amp;", "&", "&lt;", "<")
+)
+

 // telegramMarkdownEntities converts the Markdown of a reply to plain text and
 // Telegram message entities. It knows bold (** and __), italic (* and _),
 // strikethrough (~~), inline code, fenced code blocks with an optional
 // telegramMarkdownEntities converts the Markdown of a reply to plain text and
 // Telegram message entities. It knows bold (** and __), italic (* and _),
 // strikethrough (~~), inline code, fenced code blocks with an optional


@@ -68,35 +77,21 @@ func telegramMarkdownEntities(markdown string) (string, []TelegramEntity) {
        return out.text.String(), out.entities
 }
 
        return out.text.String(), out.entities
 }
 

-// telegramEscapeTags puts a backslash before each < of markdown outside inline
-// code and fenced code blocks, so that rich Markdown shows a tag name as text
-// and reads no raw HTML. A < that a backslash already makes plain stays as it
-// is. A code block that does not close runs to the end, as in Markdown.
+// telegramEscapeTags writes each & of markdown outside inline code and fenced
+// code blocks as &amp; and each < as &lt;. Rich Markdown reads raw HTML and
+// HTML entities in prose, and it does not read a backslash before < as an
+// escape. With the entities, it shows a tag name and an entity that the model
+// wrote in prose as text.

 func telegramEscapeTags(markdown string) string {
 func telegramEscapeTags(markdown string) string {

-       var out strings.Builder
-       lines := strings.Split(markdown, "\n")
-       for index := 0; index < len(lines); index++ {
-               if index > 0 {
-                       out.WriteString("\n")
-               }
-
-               line := lines[index]
-               _, fence := telegramCodeFence(line)
-               if fence == "" {
-                       telegramEscapeLineTags(&out, line)
-                       continue
-               }
-
-               end := telegramClosingFence(lines, index, fence)
-               if end < 0 {
-                       end = len(lines) - 1
-               }
-
-               out.WriteString(strings.Join(lines[index:end+1], "\n"))
-               index = end
-       }
+       return telegramRewriteProse(markdown, telegramTagEscaper.Replace)
+}

 
 

-       return out.String()
+// telegramUnescapeTags gives back the Markdown that telegramEscapeTags escaped.
+// It is exact, because after the escape each & outside code starts &amp; or
+// &lt;. It also works on a part of telegramSplitRich, because a part has the
+// same code as the text that it comes from.
+func telegramUnescapeTags(markdown string) string {
+       return telegramRewriteProse(markdown, telegramTagUnescaper.Replace)

 }
 
 // telegramCodeFence returns the language of a line that opens a fenced code
 }
 
 // telegramCodeFence returns the language of a line that opens a fenced code


@@ -256,17 +251,46 @@ func telegramHeadingText(line string) string {
        return rest
 }
 
        return rest
 }
 

-// telegramEscapeLineTags writes line to out with a backslash before each < that
-// is outside inline code and not already plain. A run of backticks that opens
-// no inline code stays as text whole, as in Markdown.
-func telegramEscapeLineTags(out *strings.Builder, line string) {
+// telegramRewriteProse returns markdown with rewrite applied to each part of a
+// line outside inline code and fenced code blocks. A code block that does not
+// close runs to the end, as in Markdown.
+func telegramRewriteProse(markdown string, rewrite func(string) string) string {
+       var out strings.Builder
+       lines := strings.Split(markdown, "\n")
+       for index := 0; index < len(lines); index++ {
+               if index > 0 {
+                       out.WriteString("\n")
+               }
+
+               line := lines[index]
+               _, fence := telegramCodeFence(line)
+               if fence == "" {
+                       telegramRewriteLineProse(&out, line, rewrite)
+                       continue
+               }
+
+               end := telegramClosingFence(lines, index, fence)
+               if end < 0 {
+                       end = len(lines) - 1
+               }
+
+               out.WriteString(strings.Join(lines[index:end+1], "\n"))
+               index = end
+       }
+
+       return out.String()
+}
+
+// telegramRewriteLineProse writes line to out with rewrite applied to each part
+// outside inline code. A backtick after a backslash opens no code. A run of
+// backticks that opens no inline code stays as text whole, as in Markdown.
+func telegramRewriteLineProse(out *strings.Builder, line string, rewrite func(string) string) {
+       prose := 0

        for index := 0; index < len(line); {
                switch line[index] {
                case '\\':
                        if index+1 < len(line) && strings.IndexByte(telegramMarkdownPunctuation, line[index+1]) >= 0 {
        for index := 0; index < len(line); {
                switch line[index] {
                case '\\':
                        if index+1 < len(line) && strings.IndexByte(telegramMarkdownPunctuation, line[index+1]) >= 0 {

-                               out.WriteString(line[index : index+2])

                                index += 2
                                index += 2

-

                                continue
                        }
                case '`':
                                continue
                        }
                case '`':


@@ -276,17 +300,18 @@ func telegramEscapeLineTags(out *strings.Builder, line string) {
                                end = span.end
                        }
 
                                end = span.end
                        }
 

+                       out.WriteString(rewrite(line[prose:index]))

                        out.WriteString(line[index:end])
                        index = end
                        out.WriteString(line[index:end])
                        index = end

+                       prose = end

 
                        continue
 
                        continue

-               case '<':
-                       out.WriteByte('\\')

                }
 
                }
 

-               out.WriteByte(line[index])

                index++
        }
                index++
        }

+
+       out.WriteString(rewrite(line[prose:]))

 }
 
 // telegramMarkSpan returns the inline mark that starts at index of line and
 }
 
 // telegramMarkSpan returns the inline mark that starts at index of line and

diff --git a/telegram_markdown_test.go b/telegram_markdown_test.go
index d80d57547de8fe24a86d07af40b4cf518d463daf..98b1f75758f02f157f49b86802cd3366ee807846 100644 (file)
--- a/telegram_markdown_test.go
+++ b/telegram_markdown_test.go
@@ -249,42 +249,52 @@ func TestEscapeTags(t *testing.T) {
                {
                        name:     "tag in prose shows as text",
                        markdown: "Use a <details> tag for the rest.",
                {
                        name:     "tag in prose shows as text",
                        markdown: "Use a <details> tag for the rest.",

-                       escaped:  `Use a \<details> tag for the rest.`,
+                       escaped:  "Use a &lt;details> tag for the rest.",

                },
                {
                        name:     "tag in inline code stays as written",
                },
                {
                        name:     "tag in inline code stays as written",

-                       markdown: "Use `<details>` and `` a<b ``, not <b>.",
-                       escaped:  "Use `<details>` and `` a<b ``, not \\<b>.",
+                       markdown: "Use `<details>` and `` a<b && c ``, not <b>.",
+                       escaped:  "Use `<details>` and `` a<b && c ``, not &lt;b>.",

                },
                {
                        name:     "tag in a code block stays as written",
                },
                {
                        name:     "tag in a code block stays as written",

-                       markdown: "<p>\n\n```html\n<details>\n```\n\n<p>",
-                       escaped:  "\\<p>\n\n```html\n<details>\n```\n\n\\<p>",
+                       markdown: "<p>\n\n```html\n<details>&amp;\n```\n\n<p>",
+                       escaped:  "&lt;p>\n\n```html\n<details>&amp;\n```\n\n&lt;p>",

                },
                {
                        name:     "code block that does not close stays as written",
                        markdown: "a <b>\n```\n<details>",
                },
                {
                        name:     "code block that does not close stays as written",
                        markdown: "a <b>\n```\n<details>",

-                       escaped:  "a \\<b>\n```\n<details>",
+                       escaped:  "a &lt;b>\n```\n<details>",

                },
                {
                        name:     "tag in a table cell shows as text",
                        markdown: "| Tag | Use |\n|-----|-----|\n| <br> | `<br>` |",
                },
                {
                        name:     "tag in a table cell shows as text",
                        markdown: "| Tag | Use |\n|-----|-----|\n| <br> | `<br>` |",

-                       escaped:  "| Tag | Use |\n|-----|-----|\n| \\<br> | `<br>` |",
+                       escaped:  "| Tag | Use |\n|-----|-----|\n| &lt;br> | `<br>` |",

                },
                {
                        name:     "autolink shows as text",
                        markdown: "See <https://example.com>.",
                },
                {
                        name:     "autolink shows as text",
                        markdown: "See <https://example.com>.",

-                       escaped:  `See \<https://example.com>.`,
+                       escaped:  "See &lt;https://example.com>.",

                },
                {
                },
                {

-                       name:     "backtick run that opens no code stays as text",
-                       markdown: "a ``b` <c>",
-                       escaped:  "a ``b` \\<c>",
+                       name:     "entity in prose shows as written",
+                       markdown: "Write &lt; or &amp; for R&D.",
+                       escaped:  "Write &amp;lt; or &amp;amp; for R&amp;D.",
+               },
+               {
+                       name:     "backslash before a bracket gets no special case",
+                       markdown: `a \<b> and \&c`,
+                       escaped:  `a \&lt;b> and \&amp;c`,

                },
                {
                },
                {

-                       name:     "escaped bracket stays as written",
-                       markdown: `a \<b> and \\<c>`,
-                       escaped:  `a \<b> and \\\<c>`,
+                       name:     "backtick after a backslash opens no code",
+                       markdown: "a \\`<b>\\` c",
+                       escaped:  "a \\`&lt;b>\\` c",
+               },
+               {
+                       name:     "backtick run that opens no code stays as text",
+                       markdown: "a ``b` <c>",
+                       escaped:  "a ``b` &lt;c>",

                },
                {
                        name:     "dollar, pipe, and equals signs stay as written",
                },
                {
                        name:     "dollar, pipe, and equals signs stay as written",


@@ -296,6 +306,7 @@ func TestEscapeTags(t *testing.T) {
        for _, tc := range testCases {
                t.Run(tc.name, func(t *testing.T) {
                        require.Equal(t, tc.escaped, telegramEscapeTags(tc.markdown))
        for _, tc := range testCases {
                t.Run(tc.name, func(t *testing.T) {
                        require.Equal(t, tc.escaped, telegramEscapeTags(tc.markdown))

+                       require.Equal(t, tc.markdown, telegramUnescapeTags(tc.escaped))

                })
        }
 }
                })
        }
 }

diff --git a/telegram_test.go b/telegram_test.go
index cf46106874d3d4d641e6a663158216f93e45dfb4..81e6e18f2299917dc03797d1d757e1757ce5d5f7 100644 (file)
--- a/telegram_test.go
+++ b/telegram_test.go
@@ -398,12 +398,12 @@ func TestReplyTagsGoAsText(t *testing.T) {
        }{
                {
                        name: "rich message escapes the tag outside code",
        }{
                {
                        name: "rich message escapes the tag outside code",

-                       sent: []string{"a \\<details> tag and `<b>`"},
+                       sent: []string{"a &lt;details> &amp; tag and `<b>`"},

                },
                {
                        name:   "entity fallback shows the tag as written",
                        status: http.StatusBadRequest,
                },
                {
                        name:   "entity fallback shows the tag as written",
                        status: http.StatusBadRequest,

-                       sent:   []string{"a <details> tag and <b>"},
+                       sent:   []string{"a <details> & tag and <b>"},

                },
        }
 
                },
        }
 


@@ -421,7 +421,7 @@ func TestReplyTagsGoAsText(t *testing.T) {
                        bot, _ := telegramStartTestBot(t, telegram)
                        bot.post(t.Context(), Event{
                                Kind: "reply",
                        bot, _ := telegramStartTestBot(t, telegram)
                        bot.post(t.Context(), Event{
                                Kind: "reply",

-                               Text: "a <details> tag and `<b>`",
+                               Text: "a <details> & tag and `<b>`",

                        })
 
                        require.Equal(t, tc.sent, telegram.Sent())
                        })
 
                        require.Equal(t, tc.sent, telegram.Sent())


@@ -451,6 +451,50 @@ func TestLongReplySplitCountsTheEscapedTags(t *testing.T) {
        }
 }
 
        }
 }
 

+func TestRefusedRichPartGoesWithTheTextAsWritten(t *testing.T) {
+       paragraph := "A <b> & c.\n\n"
+       reply := strings.Repeat(paragraph, telegramRichBytes/len(paragraph))
+
+       telegram := newTelegramFake(t)
+       refused := false
+       telegram.fail = func(call TelegramFakeCall) (int, string) {
+               if call.method != "sendRichMessage" || refused {
+                       return 0, ""
+               }
+
+               refused = true
+
+               return http.StatusBadRequest, "Bad Request"
+       }
+
+       bot, _ := telegramStartTestBot(t, telegram)
+       bot.post(t.Context(), Event{
+               Kind: "reply",
+               Text: reply,
+       })
+
+       var fallback, rich []string
+       for _, call := range telegram.Calls() {
+               switch {
+               case !call.ok:
+               case call.method == "sendMessage":
+                       fallback = append(fallback, call.params["text"].(string))
+               case call.method == "sendRichMessage":
+                       rich = append(rich, call.params["rich_message"].(map[string]any)["markdown"].(string))
+               }
+       }
+
+       require.Greater(t, len(fallback), 1)
+       require.Len(t, rich, 1)
+       require.True(t, strings.HasPrefix(reply, strings.Join(fallback, "")), "fallback text is not the reply as written")
+
+       for _, text := range fallback {
+               require.LessOrEqual(t, telegramUTF16Units(text), telegramMessageUnits)
+       }
+
+       require.True(t, reply == strings.Join(fallback, "")+telegramUnescapeTags(rich[0]), "reply is not sent once and whole")
+}
+

 func TestRefusedEntitiesGoAgainAsPlainText(t *testing.T) {
        telegram := newTelegramFake(t)
        telegram.fail = func(call TelegramFakeCall) (int, string) {
 func TestRefusedEntitiesGoAgainAsPlainText(t *testing.T) {
        telegram := newTelegramFake(t)
        telegram.fail = func(call TelegramFakeCall) (int, string) {


@@ -629,6 +673,12 @@ func TestSplitRich(t *testing.T) {
                        limit:    3,
                        parts:    []string{"ab", "\u00e9c", "d"},
                },
                        limit:    3,
                        parts:    []string{"ab", "\u00e9c", "d"},
                },

+               {
+                       name:     "long line is not cut inside an entity",
+                       markdown: "abc&amp;def",
+                       limit:    6,
+                       parts:    []string{"abc", "&amp;d", "ef"},
+               },

        }
 
        for _, tc := range testCases {
        }
 
        for _, tc := range testCases {