From: Anderson A. Giacomolli Date: Sat, 7 Nov 2026 19:38:29 +0000 (-0300) Subject: Escape tags in rich replies as HTML entities X-Git-Url: http://git.giacomolli.uk/onechat/commitdiff_plain/5bb22ac2fc501bad7327c119f1d3ac514e3d5398 Escape tags in rich replies as HTML entities The bot writes each < of a reply outside inline code and fenced code blocks as < and each & as & before it sends the reply with sendRichMessage. Rich Markdown does not read a backslash before < as an escape: it shows the backslash and still reads the tag as HTML. It reads HTML entities, so < shows as <. The escape of & makes an entity that the model writes in prose, such as <, show as written. The entity fallback reads no HTML, so a rich part that Telegram refuses goes to the fallback with the escape reversed, and the user sees the text as the model wrote it. The reverse is exact, because after the escape each & outside code starts an entity of the escape. The escape and its reverse find the code of a text in the same way, so the reverse also works on each part of the split. The split still counts the escaped text against the rich limit, and the fallback splits the text without the escape at its own limit. When the split cuts a line longer than the limit, it cuts before an & in the last four bytes, so that it does not cut an entity. Issue: d75e312 --- diff --git a/telegram.go b/telegram.go index 13c200c..6d5332d 100644 --- a/telegram.go +++ b/telegram.go @@ -311,9 +311,9 @@ func (b *TelegramBot) typing(ctx context.Context) { } // post sends event to the chat. A reply goes as rich messages of its Markdown, -// with each < outside code escaped by telegramEscapeTags and then split with -// telegramSplitRich, so that the length limit counts the escaped text. Every -// other event goes as plain text, split with telegramSplitMessage. +// with each < and & outside code escaped by telegramEscapeTags and then split +// with telegramSplitRich, so that the length limit counts the escaped text. +// Every other event goes as plain text, split with telegramSplitMessage. func (b *TelegramBot) post(ctx context.Context, event Event) { if event.Kind != "reply" { for _, text := range telegramSplitMessage(telegramText(event), telegramMessageUnits) { @@ -411,11 +411,13 @@ func (b *TelegramBot) deliver(ctx context.Context, part TelegramPart) { } } -// deliverRich sends markdown to the chat as a rich message. When the Bot API -// refuses it with HTTP 400 or 404, the message was not sent, so deliverRich -// sends the same Markdown once with message entities, and the reply is not -// lost and does not repeat. It tries again after each failure that a later try -// can fix, until ctx ends. It drops the message after any other failure. +// deliverRich sends markdown, escaped with telegramEscapeTags, to the chat as a +// rich message. When the Bot API refuses it with HTTP 400 or 404, the message +// was not sent, so deliverRich sends the text of the part once with message +// entities, and the reply is not lost and does not repeat. Message entities +// read no HTML, so the fallback gets the Markdown of telegramUnescapeTags, as +// the model wrote it. It tries again after each failure that a later try can +// fix, until ctx ends. It drops the message after any other failure. func (b *TelegramBot) deliverRich(ctx context.Context, markdown string) { wait := b.minWait for { @@ -426,7 +428,7 @@ func (b *TelegramBot) deliverRich(ctx context.Context, markdown string) { if telegramRejected(err) { b.logger.Printf("rich message refused, the reply goes with entities: %v", err) - for _, part := range telegramReplyParts(markdown) { + for _, part := range telegramReplyParts(telegramUnescapeTags(markdown)) { b.deliver(ctx, part) } @@ -546,7 +548,8 @@ func telegramText(event Event) string { // after the last line break outside a code block, then after the last line // break in a code block, which it closes at the end of the part and opens again // in the next part with the same fence line. A line longer than limit is cut at -// the limit. It never cuts a character. +// the limit, or before an & in the four bytes before the limit, so that it does +// not cut an entity of telegramEscapeTags. It never cuts a character. func telegramSplitRich(markdown string, limit int) []string { var parts []string for len(markdown) > limit { @@ -649,6 +652,12 @@ func telegramRichCut(markdown string, limit int) (int, string, string) { cut-- } + start := max(cut-len("&")+1, 0) + entity := strings.LastIndexByte(markdown[start:cut], '&') + if entity >= 0 && start+entity > 0 { + cut = start + entity + } + return cut, "", "" } diff --git a/telegram_markdown.go b/telegram_markdown.go index 16b83f2..8738bcd 100644 --- a/telegram_markdown.go +++ b/telegram_markdown.go @@ -11,6 +11,15 @@ import ( // plain. const telegramMarkdownPunctuation = "!\"#$%&'()*+,-./:;<=>?@[\\]^_`{|}~" +var ( + // telegramTagEscaper writes the characters that rich Markdown reads as HTML + // as HTML entities. + telegramTagEscaper = strings.NewReplacer("&", "&", "<", "<") + + // telegramTagUnescaper reverses telegramTagEscaper. + telegramTagUnescaper = strings.NewReplacer("&", "&", "<", "<") +) + // telegramMarkdownEntities converts the Markdown of a reply to plain text and // Telegram message entities. It knows bold (** and __), italic (* and _), // strikethrough (~~), inline code, fenced code blocks with an optional @@ -68,35 +77,21 @@ func telegramMarkdownEntities(markdown string) (string, []TelegramEntity) { return out.text.String(), out.entities } -// telegramEscapeTags puts a backslash before each < of markdown outside inline -// code and fenced code blocks, so that rich Markdown shows a tag name as text -// and reads no raw HTML. A < that a backslash already makes plain stays as it -// is. A code block that does not close runs to the end, as in Markdown. +// telegramEscapeTags writes each & of markdown outside inline code and fenced +// code blocks as & and each < as <. Rich Markdown reads raw HTML and +// HTML entities in prose, and it does not read a backslash before < as an +// escape. With the entities, it shows a tag name and an entity that the model +// wrote in prose as text. func telegramEscapeTags(markdown string) string { - var out strings.Builder - lines := strings.Split(markdown, "\n") - for index := 0; index < len(lines); index++ { - if index > 0 { - out.WriteString("\n") - } - - line := lines[index] - _, fence := telegramCodeFence(line) - if fence == "" { - telegramEscapeLineTags(&out, line) - continue - } - - end := telegramClosingFence(lines, index, fence) - if end < 0 { - end = len(lines) - 1 - } - - out.WriteString(strings.Join(lines[index:end+1], "\n")) - index = end - } + return telegramRewriteProse(markdown, telegramTagEscaper.Replace) +} - return out.String() +// telegramUnescapeTags gives back the Markdown that telegramEscapeTags escaped. +// It is exact, because after the escape each & outside code starts & or +// <. It also works on a part of telegramSplitRich, because a part has the +// same code as the text that it comes from. +func telegramUnescapeTags(markdown string) string { + return telegramRewriteProse(markdown, telegramTagUnescaper.Replace) } // telegramCodeFence returns the language of a line that opens a fenced code @@ -256,17 +251,46 @@ func telegramHeadingText(line string) string { return rest } -// telegramEscapeLineTags writes line to out with a backslash before each < that -// is outside inline code and not already plain. A run of backticks that opens -// no inline code stays as text whole, as in Markdown. -func telegramEscapeLineTags(out *strings.Builder, line string) { +// telegramRewriteProse returns markdown with rewrite applied to each part of a +// line outside inline code and fenced code blocks. A code block that does not +// close runs to the end, as in Markdown. +func telegramRewriteProse(markdown string, rewrite func(string) string) string { + var out strings.Builder + lines := strings.Split(markdown, "\n") + for index := 0; index < len(lines); index++ { + if index > 0 { + out.WriteString("\n") + } + + line := lines[index] + _, fence := telegramCodeFence(line) + if fence == "" { + telegramRewriteLineProse(&out, line, rewrite) + continue + } + + end := telegramClosingFence(lines, index, fence) + if end < 0 { + end = len(lines) - 1 + } + + out.WriteString(strings.Join(lines[index:end+1], "\n")) + index = end + } + + return out.String() +} + +// telegramRewriteLineProse writes line to out with rewrite applied to each part +// outside inline code. A backtick after a backslash opens no code. A run of +// backticks that opens no inline code stays as text whole, as in Markdown. +func telegramRewriteLineProse(out *strings.Builder, line string, rewrite func(string) string) { + prose := 0 for index := 0; index < len(line); { switch line[index] { case '\\': if index+1 < len(line) && strings.IndexByte(telegramMarkdownPunctuation, line[index+1]) >= 0 { - out.WriteString(line[index : index+2]) index += 2 - continue } case '`': @@ -276,17 +300,18 @@ func telegramEscapeLineTags(out *strings.Builder, line string) { end = span.end } + out.WriteString(rewrite(line[prose:index])) out.WriteString(line[index:end]) index = end + prose = end continue - case '<': - out.WriteByte('\\') } - out.WriteByte(line[index]) index++ } + + out.WriteString(rewrite(line[prose:])) } // telegramMarkSpan returns the inline mark that starts at index of line and diff --git a/telegram_markdown_test.go b/telegram_markdown_test.go index d80d575..98b1f75 100644 --- a/telegram_markdown_test.go +++ b/telegram_markdown_test.go @@ -249,42 +249,52 @@ func TestEscapeTags(t *testing.T) { { name: "tag in prose shows as text", markdown: "Use a
tag for the rest.", - escaped: `Use a \
tag for the rest.`, + escaped: "Use a <details> tag for the rest.", }, { name: "tag in inline code stays as written", - markdown: "Use `
` and `` a.", - escaped: "Use `
` and `` a.", + markdown: "Use `
` and `` a.", + escaped: "Use `
` and `` a.", }, { name: "tag in a code block stays as written", - markdown: "

\n\n```html\n

\n```\n\n

", - escaped: "\\

\n\n```html\n

\n```\n\n\\

", + markdown: "

\n\n```html\n

&\n```\n\n

", + escaped: "<p>\n\n```html\n

&\n```\n\n<p>", }, { name: "code block that does not close stays as written", markdown: "a \n```\n
", - escaped: "a \\\n```\n
", + escaped: "a <b>\n```\n
", }, { name: "tag in a table cell shows as text", markdown: "| Tag | Use |\n|-----|-----|\n|
| `
` |", - escaped: "| Tag | Use |\n|-----|-----|\n| \\
| `
` |", + escaped: "| Tag | Use |\n|-----|-----|\n| <br> | `
` |", }, { name: "autolink shows as text", markdown: "See .", - escaped: `See \.`, + escaped: "See <https://example.com>.", }, { - name: "backtick run that opens no code stays as text", - markdown: "a ``b` ", - escaped: "a ``b` \\", + name: "entity in prose shows as written", + markdown: "Write < or & for R&D.", + escaped: "Write &lt; or &amp; for R&D.", + }, + { + name: "backslash before a bracket gets no special case", + markdown: `a \ and \&c`, + escaped: `a \<b> and \&c`, }, { - name: "escaped bracket stays as written", - markdown: `a \ and \\`, - escaped: `a \ and \\\`, + name: "backtick after a backslash opens no code", + markdown: "a \\`\\` c", + escaped: "a \\`<b>\\` c", + }, + { + name: "backtick run that opens no code stays as text", + markdown: "a ``b` ", + escaped: "a ``b` <c>", }, { name: "dollar, pipe, and equals signs stay as written", @@ -296,6 +306,7 @@ func TestEscapeTags(t *testing.T) { for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { require.Equal(t, tc.escaped, telegramEscapeTags(tc.markdown)) + require.Equal(t, tc.markdown, telegramUnescapeTags(tc.escaped)) }) } } diff --git a/telegram_test.go b/telegram_test.go index cf46106..81e6e18 100644 --- a/telegram_test.go +++ b/telegram_test.go @@ -398,12 +398,12 @@ func TestReplyTagsGoAsText(t *testing.T) { }{ { name: "rich message escapes the tag outside code", - sent: []string{"a \\
tag and ``"}, + sent: []string{"a <details> & tag and ``"}, }, { name: "entity fallback shows the tag as written", status: http.StatusBadRequest, - sent: []string{"a
tag and "}, + sent: []string{"a
& tag and "}, }, } @@ -421,7 +421,7 @@ func TestReplyTagsGoAsText(t *testing.T) { bot, _ := telegramStartTestBot(t, telegram) bot.post(t.Context(), Event{ Kind: "reply", - Text: "a
tag and ``", + Text: "a
& tag and ``", }) require.Equal(t, tc.sent, telegram.Sent()) @@ -451,6 +451,50 @@ func TestLongReplySplitCountsTheEscapedTags(t *testing.T) { } } +func TestRefusedRichPartGoesWithTheTextAsWritten(t *testing.T) { + paragraph := "A & c.\n\n" + reply := strings.Repeat(paragraph, telegramRichBytes/len(paragraph)) + + telegram := newTelegramFake(t) + refused := false + telegram.fail = func(call TelegramFakeCall) (int, string) { + if call.method != "sendRichMessage" || refused { + return 0, "" + } + + refused = true + + return http.StatusBadRequest, "Bad Request" + } + + bot, _ := telegramStartTestBot(t, telegram) + bot.post(t.Context(), Event{ + Kind: "reply", + Text: reply, + }) + + var fallback, rich []string + for _, call := range telegram.Calls() { + switch { + case !call.ok: + case call.method == "sendMessage": + fallback = append(fallback, call.params["text"].(string)) + case call.method == "sendRichMessage": + rich = append(rich, call.params["rich_message"].(map[string]any)["markdown"].(string)) + } + } + + require.Greater(t, len(fallback), 1) + require.Len(t, rich, 1) + require.True(t, strings.HasPrefix(reply, strings.Join(fallback, "")), "fallback text is not the reply as written") + + for _, text := range fallback { + require.LessOrEqual(t, telegramUTF16Units(text), telegramMessageUnits) + } + + require.True(t, reply == strings.Join(fallback, "")+telegramUnescapeTags(rich[0]), "reply is not sent once and whole") +} + func TestRefusedEntitiesGoAgainAsPlainText(t *testing.T) { telegram := newTelegramFake(t) telegram.fail = func(call TelegramFakeCall) (int, string) { @@ -629,6 +673,12 @@ func TestSplitRich(t *testing.T) { limit: 3, parts: []string{"ab", "\u00e9c", "d"}, }, + { + name: "long line is not cut inside an entity", + markdown: "abc&def", + limit: 6, + parts: []string{"abc", "&d", "ef"}, + }, } for _, tc := range testCases {