}
// post sends event to the chat. A reply goes as rich messages of its Markdown,
-// with each < outside code escaped by telegramEscapeTags and then split with
-// telegramSplitRich, so that the length limit counts the escaped text. Every
-// other event goes as plain text, split with telegramSplitMessage.
+// with each < and & outside code escaped by telegramEscapeTags and then split
+// with telegramSplitRich, so that the length limit counts the escaped text.
+// Every other event goes as plain text, split with telegramSplitMessage.
func (b *TelegramBot) post(ctx context.Context, event Event) {
if event.Kind != "reply" {
for _, text := range telegramSplitMessage(telegramText(event), telegramMessageUnits) {
}
}
-// deliverRich sends markdown to the chat as a rich message. When the Bot API
-// refuses it with HTTP 400 or 404, the message was not sent, so deliverRich
-// sends the same Markdown once with message entities, and the reply is not
-// lost and does not repeat. It tries again after each failure that a later try
-// can fix, until ctx ends. It drops the message after any other failure.
+// deliverRich sends markdown, escaped with telegramEscapeTags, to the chat as a
+// rich message. When the Bot API refuses it with HTTP 400 or 404, the message
+// was not sent, so deliverRich sends the text of the part once with message
+// entities, and the reply is not lost and does not repeat. Message entities
+// read no HTML, so the fallback gets the Markdown of telegramUnescapeTags, as
+// the model wrote it. It tries again after each failure that a later try can
+// fix, until ctx ends. It drops the message after any other failure.
func (b *TelegramBot) deliverRich(ctx context.Context, markdown string) {
wait := b.minWait
for {
if telegramRejected(err) {
b.logger.Printf("rich message refused, the reply goes with entities: %v", err)
- for _, part := range telegramReplyParts(markdown) {
+ for _, part := range telegramReplyParts(telegramUnescapeTags(markdown)) {
b.deliver(ctx, part)
}
// after the last line break outside a code block, then after the last line
// break in a code block, which it closes at the end of the part and opens again
// in the next part with the same fence line. A line longer than limit is cut at
-// the limit. It never cuts a character.
+// the limit, or before an & in the four bytes before the limit, so that it does
+// not cut an entity of telegramEscapeTags. It never cuts a character.
func telegramSplitRich(markdown string, limit int) []string {
var parts []string
for len(markdown) > limit {
cut--
}
+ start := max(cut-len("&")+1, 0)
+ entity := strings.LastIndexByte(markdown[start:cut], '&')
+ if entity >= 0 && start+entity > 0 {
+ cut = start + entity
+ }
+
return cut, "", ""
}
// plain.
const telegramMarkdownPunctuation = "!\"#$%&'()*+,-./:;<=>?@[\\]^_`{|}~"
+var (
+ // telegramTagEscaper writes the characters that rich Markdown reads as HTML
+ // as HTML entities.
+ telegramTagEscaper = strings.NewReplacer("&", "&", "<", "<")
+
+ // telegramTagUnescaper reverses telegramTagEscaper.
+ telegramTagUnescaper = strings.NewReplacer("&", "&", "<", "<")
+)
+
// telegramMarkdownEntities converts the Markdown of a reply to plain text and
// Telegram message entities. It knows bold (** and __), italic (* and _),
// strikethrough (~~), inline code, fenced code blocks with an optional
return out.text.String(), out.entities
}
-// telegramEscapeTags puts a backslash before each < of markdown outside inline
-// code and fenced code blocks, so that rich Markdown shows a tag name as text
-// and reads no raw HTML. A < that a backslash already makes plain stays as it
-// is. A code block that does not close runs to the end, as in Markdown.
+// telegramEscapeTags writes each & of markdown outside inline code and fenced
+// code blocks as & and each < as <. Rich Markdown reads raw HTML and
+// HTML entities in prose, and it does not read a backslash before < as an
+// escape. With the entities, it shows a tag name and an entity that the model
+// wrote in prose as text.
func telegramEscapeTags(markdown string) string {
- var out strings.Builder
- lines := strings.Split(markdown, "\n")
- for index := 0; index < len(lines); index++ {
- if index > 0 {
- out.WriteString("\n")
- }
-
- line := lines[index]
- _, fence := telegramCodeFence(line)
- if fence == "" {
- telegramEscapeLineTags(&out, line)
- continue
- }
-
- end := telegramClosingFence(lines, index, fence)
- if end < 0 {
- end = len(lines) - 1
- }
-
- out.WriteString(strings.Join(lines[index:end+1], "\n"))
- index = end
- }
+ return telegramRewriteProse(markdown, telegramTagEscaper.Replace)
+}
- return out.String()
+// telegramUnescapeTags gives back the Markdown that telegramEscapeTags escaped.
+// It is exact, because after the escape each & outside code starts & or
+// <. It also works on a part of telegramSplitRich, because a part has the
+// same code as the text that it comes from.
+func telegramUnescapeTags(markdown string) string {
+ return telegramRewriteProse(markdown, telegramTagUnescaper.Replace)
}
// telegramCodeFence returns the language of a line that opens a fenced code
return rest
}
-// telegramEscapeLineTags writes line to out with a backslash before each < that
-// is outside inline code and not already plain. A run of backticks that opens
-// no inline code stays as text whole, as in Markdown.
-func telegramEscapeLineTags(out *strings.Builder, line string) {
+// telegramRewriteProse returns markdown with rewrite applied to each part of a
+// line outside inline code and fenced code blocks. A code block that does not
+// close runs to the end, as in Markdown.
+func telegramRewriteProse(markdown string, rewrite func(string) string) string {
+ var out strings.Builder
+ lines := strings.Split(markdown, "\n")
+ for index := 0; index < len(lines); index++ {
+ if index > 0 {
+ out.WriteString("\n")
+ }
+
+ line := lines[index]
+ _, fence := telegramCodeFence(line)
+ if fence == "" {
+ telegramRewriteLineProse(&out, line, rewrite)
+ continue
+ }
+
+ end := telegramClosingFence(lines, index, fence)
+ if end < 0 {
+ end = len(lines) - 1
+ }
+
+ out.WriteString(strings.Join(lines[index:end+1], "\n"))
+ index = end
+ }
+
+ return out.String()
+}
+
+// telegramRewriteLineProse writes line to out with rewrite applied to each part
+// outside inline code. A backtick after a backslash opens no code. A run of
+// backticks that opens no inline code stays as text whole, as in Markdown.
+func telegramRewriteLineProse(out *strings.Builder, line string, rewrite func(string) string) {
+ prose := 0
for index := 0; index < len(line); {
switch line[index] {
case '\\':
if index+1 < len(line) && strings.IndexByte(telegramMarkdownPunctuation, line[index+1]) >= 0 {
- out.WriteString(line[index : index+2])
index += 2
-
continue
}
case '`':
end = span.end
}
+ out.WriteString(rewrite(line[prose:index]))
out.WriteString(line[index:end])
index = end
+ prose = end
continue
- case '<':
- out.WriteByte('\\')
}
- out.WriteByte(line[index])
index++
}
+
+ out.WriteString(rewrite(line[prose:]))
}
// telegramMarkSpan returns the inline mark that starts at index of line and
{
name: "tag in prose shows as text",
markdown: "Use a <details> tag for the rest.",
- escaped: `Use a \<details> tag for the rest.`,
+ escaped: "Use a <details> tag for the rest.",
},
{
name: "tag in inline code stays as written",
- markdown: "Use `<details>` and `` a<b ``, not <b>.",
- escaped: "Use `<details>` and `` a<b ``, not \\<b>.",
+ markdown: "Use `<details>` and `` a<b && c ``, not <b>.",
+ escaped: "Use `<details>` and `` a<b && c ``, not <b>.",
},
{
name: "tag in a code block stays as written",
- markdown: "<p>\n\n```html\n<details>\n```\n\n<p>",
- escaped: "\\<p>\n\n```html\n<details>\n```\n\n\\<p>",
+ markdown: "<p>\n\n```html\n<details>
&\n```\n\n<p>",
+ escaped: "<p>\n\n```html\n<details>&\n```\n\n<p>",
},
{
name: "code block that does not close stays as written",
markdown: "a <b>\n```\n<details>",
- escaped: "a \\<b>\n```\n<details>",
+ escaped: "a <b>\n```\n<details>",
},
{
name: "tag in a table cell shows as text",
markdown: "| Tag | Use |\n|-----|-----|\n| <br> | `<br>` |",
- escaped: "| Tag | Use |\n|-----|-----|\n| \\<br> | `<br>` |",
+ escaped: "| Tag | Use |\n|-----|-----|\n| <br> | `<br>` |",
},
{
name: "autolink shows as text",
markdown: "See <https://example.com>.",
- escaped: `See \<https://example.com>.`,
+ escaped: "See <https://example.com>.",
},
{
- name: "backtick run that opens no code stays as text",
- markdown: "a ``b` <c>",
- escaped: "a ``b` \\<c>",
+ name: "entity in prose shows as written",
+ markdown: "Write < or & for R&D.",
+ escaped: "Write &lt; or &amp; for R&D.",
+ },
+ {
+ name: "backslash before a bracket gets no special case",
+ markdown: `a \<b> and \&c`,
+ escaped: `a \<b> and \&c`,
},
{
- name: "escaped bracket stays as written",
- markdown: `a \<b> and \\<c>`,
- escaped: `a \<b> and \\\<c>`,
+ name: "backtick after a backslash opens no code",
+ markdown: "a \\`<b>\\` c",
+ escaped: "a \\`<b>\\` c",
+ },
+ {
+ name: "backtick run that opens no code stays as text",
+ markdown: "a ``b` <c>",
+ escaped: "a ``b` <c>",
},
{
name: "dollar, pipe, and equals signs stay as written",
for _, tc := range testCases {
t.Run(tc.name, func(t *testing.T) {
require.Equal(t, tc.escaped, telegramEscapeTags(tc.markdown))
+ require.Equal(t, tc.markdown, telegramUnescapeTags(tc.escaped))
})
}
}
}{
{
name: "rich message escapes the tag outside code",
- sent: []string{"a \\<details> tag and `<b>`"},
+ sent: []string{"a <details> & tag and `<b>`"},
},
{
name: "entity fallback shows the tag as written",
status: http.StatusBadRequest,
- sent: []string{"a <details> tag and <b>"},
+ sent: []string{"a <details> & tag and <b>"},
},
}
bot, _ := telegramStartTestBot(t, telegram)
bot.post(t.Context(), Event{
Kind: "reply",
- Text: "a <details> tag and `<b>`",
+ Text: "a <details> & tag and `<b>`",
})
require.Equal(t, tc.sent, telegram.Sent())
}
}
+func TestRefusedRichPartGoesWithTheTextAsWritten(t *testing.T) {
+ paragraph := "A <b> & c.\n\n"
+ reply := strings.Repeat(paragraph, telegramRichBytes/len(paragraph))
+
+ telegram := newTelegramFake(t)
+ refused := false
+ telegram.fail = func(call TelegramFakeCall) (int, string) {
+ if call.method != "sendRichMessage" || refused {
+ return 0, ""
+ }
+
+ refused = true
+
+ return http.StatusBadRequest, "Bad Request"
+ }
+
+ bot, _ := telegramStartTestBot(t, telegram)
+ bot.post(t.Context(), Event{
+ Kind: "reply",
+ Text: reply,
+ })
+
+ var fallback, rich []string
+ for _, call := range telegram.Calls() {
+ switch {
+ case !call.ok:
+ case call.method == "sendMessage":
+ fallback = append(fallback, call.params["text"].(string))
+ case call.method == "sendRichMessage":
+ rich = append(rich, call.params["rich_message"].(map[string]any)["markdown"].(string))
+ }
+ }
+
+ require.Greater(t, len(fallback), 1)
+ require.Len(t, rich, 1)
+ require.True(t, strings.HasPrefix(reply, strings.Join(fallback, "")), "fallback text is not the reply as written")
+
+ for _, text := range fallback {
+ require.LessOrEqual(t, telegramUTF16Units(text), telegramMessageUnits)
+ }
+
+ require.True(t, reply == strings.Join(fallback, "")+telegramUnescapeTags(rich[0]), "reply is not sent once and whole")
+}
+
func TestRefusedEntitiesGoAgainAsPlainText(t *testing.T) {
telegram := newTelegramFake(t)
telegram.fail = func(call TelegramFakeCall) (int, string) {
limit: 3,
parts: []string{"ab", "\u00e9c", "d"},
},
+ {
+ name: "long line is not cut inside an entity",
+ markdown: "abc&def",
+ limit: 6,
+ parts: []string{"abc", "&d", "ef"},
+ },
}
for _, tc := range testCases {
projects / onechat / commitdiff