X-Git-Url: https://git.giacomolli.uk/onechat/blobdiff_plain/09cde69fdffe2721434392036f98dc241177f5a1..c781f79f3c8cb9a7577ee1574d196988da2719b3:/telegram_markdown.go

diff --git a/telegram_markdown.go b/telegram_markdown.go
index 54bfbcb..b6ea5a2 100644
--- a/telegram_markdown.go
+++ b/telegram_markdown.go
@@ -3,108 +3,16 @@ package main
 import (
 	"slices"
 	"strings"
-	"unicode"
-	"unicode/utf8"
 )
 
-// telegramMarkdownPunctuation holds the ASCII characters that a backslash makes
-// plain.
-const telegramMarkdownPunctuation = "!\"#$%&'()*+,-./:;<=>?@[\\]^_`{|}~"
+var (
+	// telegramTagEscaper writes the characters that rich Markdown reads as
+	// HTML as HTML entities.
+	telegramTagEscaper = strings.NewReplacer("&", "&amp;", "<", "&lt;")
 
-// telegramMarkdownEntities converts the Markdown of a reply to plain text and
-// Telegram message entities. It knows bold (** and __), italic (* and _),
-// strikethrough (~~), inline code, fenced code blocks with an optional
-// language, links [text](url), and ATX headings, which become bold. Lists,
-// quotes, and tables stay as their text. A mark that does not close stays as
-// text with no entity, and code holds no entity.
-//
-// Inline marks do not cross a line break, so a mark that does not close cannot
-// change the rest of the reply.
-func telegramMarkdownEntities(markdown string) (string, []TelegramEntity) {
-	var out TelegramMarkdownText
-	lines := strings.Split(markdown, "\n")
-	for index := 0; index < len(lines); index++ {
-		if index > 0 {
-			out.add("\n")
-		}
-
-		line := lines[index]
-		language, fence := telegramCodeFence(line)
-		end := telegramClosingFence(lines, index, fence)
-		heading := telegramHeadingText(line)
-		switch {
-		case end > index:
-			start := out.units
-			out.add(strings.Join(lines[index+1:end], "\n"))
-			out.entity(TelegramEntity{
-				Type:     "pre",
-				Offset:   start,
-				Length:   out.units - start,
-				Language: language,
-			})
-
-			index = end
-		case heading != "":
-			start := out.units
-			out.inline(heading)
-			out.entity(TelegramEntity{
-				Type:   "bold",
-				Offset: start,
-				Length: out.units - start,
-			})
-		default:
-			out.inline(line)
-		}
-	}
-
-	slices.SortStableFunc(out.entities, func(a, b TelegramEntity) int {
-		if a.Offset != b.Offset {
-			return a.Offset - b.Offset
-		}
-
-		return b.Length - a.Length
-	})
-
-	return out.text.String(), out.entities
-}
-
-// telegramCodeFence returns the language of a line that opens a fenced code
-// block, and the fence. The fence is "" when line does not open a block.
-func telegramCodeFence(line string) (string, string) {
-	trimmed := strings.TrimLeft(line, " ")
-	if len(line)-len(trimmed) > 3 || !strings.HasPrefix(trimmed, "```") {
-		return "", ""
-	}
-
-	run := len(trimmed) - len(strings.TrimLeft(trimmed, "`"))
-	info := strings.TrimSpace(trimmed[run:])
-	if strings.Contains(info, "`") {
-		return "", ""
-	}
-
-	language, _, _ := strings.Cut(info, " ")
-
-	return language, trimmed[:run]
-}
-
-// telegramClosesFence tells if line closes the code block of fence.
-func telegramClosesFence(line, fence string) bool {
-	trimmed := strings.TrimSpace(line)
-	return strings.HasPrefix(trimmed, fence) && strings.Trim(trimmed, "`") == ""
-}
-
-// telegramUTF16Units returns the number of UTF-16 code units of text.
-func telegramUTF16Units(text string) int {
-	units := 0
-	for _, char := range text {
-		units++
-		if char > 0xFFFF {
-			units++
-		}
-	}
-
-	return units
-}
+	// telegramTagUnescaper reverses telegramTagEscaper.
+	telegramTagUnescaper = strings.NewReplacer("&amp;", "&", "&lt;", "<")
+)
 
 // TelegramMarkdownText is the plain text of a Markdown reply and its Telegram
 // entities, while the conversion builds it. Units counts the UTF-16 code units
@@ -118,6 +26,7 @@ type TelegramMarkdownText struct {
 // add appends text.
 func (m *TelegramMarkdownText) add(text string) {
 	m.text.WriteString(text)
+
 	m.units += telegramUTF16Units(text)
 }
 
@@ -136,21 +45,23 @@ func (m *TelegramMarkdownText) inline(line string) {
 	plain := 0
 	index := 0
 	for index < len(line) {
-		if line[index] == '\\' && index+1 < len(line) && strings.IndexByte(telegramMarkdownPunctuation, line[index+1]) >= 0 {
+		if line[index] == '\\' && index+1 < len(line) && strings.IndexByte(markdownPunctuation, line[index+1]) >= 0 {
 			m.add(line[plain:index])
+
 			plain = index + 1
 			index += 2
 
 			continue
 		}
 
-		span, found := telegramMarkSpan(line, index)
+		span, found := markdownMarkSpan(line, index)
 		if !found {
 			index++
 			continue
 		}
 
 		m.add(line[plain:index])
+
 		start := m.units
 		if span.kind == "code" {
 			m.add(span.content)
@@ -172,236 +83,157 @@ func (m *TelegramMarkdownText) inline(line string) {
 	m.add(line[plain:])
 }
 
-// TelegramMarkdownSpan is an inline mark of a line: the entity type, the
-// content inside the marks, the index after the closing mark, and the URL of a
-// link.
-type TelegramMarkdownSpan struct {
-	kind    string
-	content string
-	end     int
-	url     string
-}
-
-// telegramClosingFence returns the index of the line after index that closes
-// the code block of fence, or -1 when fence is "" or no line closes it.
-func telegramClosingFence(lines []string, index int, fence string) int {
-	if fence == "" {
-		return -1
-	}
+// telegramMarkdownEntities converts the Markdown of a reply to plain text and
+// Telegram message entities. It knows bold (** and __), italic (* and _),
+// strikethrough (~~), inline code, fenced code blocks with an optional
+// language, links [text](url), and ATX headings, which become bold. Lists,
+// quotes, and tables stay as their text. A mark that does not close stays as
+// text with no entity, and code holds no entity.
+//
+// Inline marks do not cross a line break, so a mark that does not close cannot
+// change the rest of the reply.
+func telegramMarkdownEntities(markdown string) (string, []TelegramEntity) {
+	var out TelegramMarkdownText
 
-	for end := index + 1; end < len(lines); end++ {
-		if telegramClosesFence(lines[end], fence) {
-			return end
+	lines := strings.Split(markdown, "\n")
+	for index := 0; index < len(lines); index++ {
+		if index > 0 {
+			out.add("\n")
 		}
-	}
-
-	return -1
-}
 
-// telegramHeadingText returns the text of an ATX heading line without its
-// marks, or "" when line is not a heading or has no text.
-func telegramHeadingText(line string) string {
-	trimmed := strings.TrimLeft(line, " ")
-	if len(line)-len(trimmed) > 3 {
-		return ""
-	}
+		line := lines[index]
+		language, fence := markdownCodeFence(line)
+		end := markdownClosingFence(lines, index, fence)
+		heading := markdownHeadingText(line)
+		switch {
+		case end > index:
+			start := out.units
+			out.add(strings.Join(lines[index+1:end], "\n"))
+			out.entity(TelegramEntity{
+				Type:     "pre",
+				Offset:   start,
+				Length:   out.units - start,
+				Language: language,
+			})
 
-	level := len(trimmed) - len(strings.TrimLeft(trimmed, "#"))
-	if level == 0 || level > 6 {
-		return ""
+			index = end
+		case heading != "":
+			start := out.units
+			out.inline(heading)
+			out.entity(TelegramEntity{
+				Type:   "bold",
+				Offset: start,
+				Length: out.units - start,
+			})
+		default:
+			out.inline(line)
+		}
 	}
 
-	rest := trimmed[level:]
-	if rest != "" && rest[0] != ' ' && rest[0] != '\t' {
-		return ""
-	}
+	slices.SortStableFunc(out.entities, func(a, b TelegramEntity) int {
+		if a.Offset != b.Offset {
+			return a.Offset - b.Offset
+		}
 
-	rest = strings.TrimSpace(rest)
-	closed := strings.TrimRight(rest, "#")
-	if closed == "" || strings.HasSuffix(closed, " ") {
-		rest = strings.TrimSpace(closed)
-	}
+		return b.Length - a.Length
+	})
 
-	return rest
+	return out.text.String(), out.entities
 }
 
-// telegramMarkSpan returns the inline mark that starts at index of line and
-// closes in line. It returns false when no mark starts there or the mark does
-// not close.
-func telegramMarkSpan(line string, index int) (TelegramMarkdownSpan, bool) {
-	switch line[index] {
-	case '`':
-		return telegramCodeSpan(line, index)
-	case '[':
-		return telegramLinkSpan(line, index)
-	case '~':
-		return telegramDelimited(line, index, "~~", "strikethrough")
-	case '*', '_':
-		double := line[index : index+1]
-		double += double
-		span, found := telegramDelimited(line, index, double, "bold")
-		if found {
-			return span, true
-		}
-
-		return telegramDelimited(line, index, line[index:index+1], "italic")
-	default:
-		return TelegramMarkdownSpan{}, false
-	}
+// telegramEscapeTags writes each & of markdown outside inline code and fenced
+// code blocks as &amp; and each < as &lt;. Rich Markdown reads raw HTML and
+// HTML entities in prose, and it does not read a backslash before < as an
+// escape. With the entities, it shows a tag name and an entity that the model
+// wrote in prose as text.
+func telegramEscapeTags(markdown string) string {
+	return telegramRewriteProse(markdown, telegramTagEscaper.Replace)
 }
 
-// telegramCodeSpan returns the inline code that starts at index: a run of
-// backticks, the code, and a run of the same length. One space on each side of
-// the code is left out, as in Markdown.
-func telegramCodeSpan(line string, index int) (TelegramMarkdownSpan, bool) {
-	run := telegramBackticks(line[index:])
-	for search := index + run; search < len(line); {
-		next := strings.IndexByte(line[search:], '`')
-		if next < 0 {
-			break
-		}
-
-		next += search
-		length := telegramBackticks(line[next:])
-		if length != run {
-			search = next + length
-			continue
-		}
-
-		content := line[index+run : next]
-		if strings.TrimSpace(content) == "" {
-			break
-		}
+// telegramUnescapeTags gives back the Markdown that telegramEscapeTags escaped.
+// It is exact, because after the escape each & outside code starts &amp; or
+// &lt;. It also works on a part of telegramSplitRich, because a part has the
+// same code as the text that it comes from.
+func telegramUnescapeTags(markdown string) string {
+	return telegramRewriteProse(markdown, telegramTagUnescaper.Replace)
+}
 
-		if len(content) > 1 && content[0] == ' ' && content[len(content)-1] == ' ' {
-			content = content[1 : len(content)-1]
+// telegramUTF16Units returns the number of UTF-16 code units of text.
+func telegramUTF16Units(text string) int {
+	units := 0
+	for _, char := range text {
+		units++
+		if char > 0xFFFF {
+			units++
 		}
-
-		return TelegramMarkdownSpan{
-			kind:    "code",
-			content: content,
-			end:     next + run,
-		}, true
 	}
 
-	return TelegramMarkdownSpan{}, false
+	return units
 }
 
-// telegramLinkSpan returns the link [text](url) that starts at index. The text
-// can hold brackets in pairs, and the URL holds no space.
-func telegramLinkSpan(line string, index int) (TelegramMarkdownSpan, bool) {
-	depth := 0
-	for closing := index; closing < len(line); closing++ {
-		switch line[closing] {
-		case '\\':
-			closing++
-		case '[':
-			depth++
-		case ']':
-			depth--
+// telegramRewriteProse returns markdown with rewrite applied to each part of a
+// line outside inline code and fenced code blocks. A code block that does not
+// close runs to the end, as in Markdown.
+func telegramRewriteProse(markdown string, rewrite func(string) string) string {
+	var out strings.Builder
+
+	lines := strings.Split(markdown, "\n")
+	for index := 0; index < len(lines); index++ {
+		if index > 0 {
+			out.WriteString("\n")
 		}
 
-		if depth > 0 {
+		line := lines[index]
+		_, fence := markdownCodeFence(line)
+		if fence == "" {
+			telegramRewriteLineProse(&out, line, rewrite)
+
 			continue
 		}
 
-		rest := line[closing+1:]
-		if closing == index+1 || !strings.HasPrefix(rest, "(") {
-			return TelegramMarkdownSpan{}, false
+		end := markdownClosingFence(lines, index, fence)
+		if end < 0 {
+			end = len(lines) - 1
 		}
 
-		url, _, found := strings.Cut(rest[1:], ")")
-		if !found || url == "" || strings.ContainsAny(url, " \t") {
-			return TelegramMarkdownSpan{}, false
-		}
+		out.WriteString(strings.Join(lines[index:end+1], "\n"))
 
-		return TelegramMarkdownSpan{
-			kind:    "text_link",
-			content: line[index+1 : closing],
-			end:     closing + 2 + len(url) + 1,
-			url:     url,
-		}, true
+		index = end
 	}
 
-	return TelegramMarkdownSpan{}, false
+	return out.String()
 }
 
-// telegramDelimited returns the span of kind between the mark at index and the
-// same mark later in line. An opening mark is followed by a character that is
-// not a space, and a closing mark follows one. A single mark is not a part of a
-// double one. A mark of underscores does not open or close inside a word, so
-// snake_case stays as text. A code span inside the content is skipped whole.
-func telegramDelimited(line string, index int, mark, kind string) (TelegramMarkdownSpan, bool) {
-	char := mark[0]
-	after := index + len(mark)
-	if !strings.HasPrefix(line[index:], mark) || after >= len(line) || telegramIsSpace(line[after]) {
-		return TelegramMarkdownSpan{}, false
-	}
-
-	if len(mark) == 1 && (line[after] == char || index > 0 && line[index-1] == char) {
-		return TelegramMarkdownSpan{}, false
-	}
-
-	if char == '_' && index > 0 && telegramWordCharBefore(line[:index]) {
-		return TelegramMarkdownSpan{}, false
-	}
-
-	for closing := after + 1; closing < len(line); closing++ {
-		switch {
-		case line[closing] == '\\':
-			closing++
-			continue
-		case line[closing] == '`':
-			span, found := telegramCodeSpan(line, closing)
+// telegramRewriteLineProse writes line to out with rewrite applied to each part
+// outside inline code. A backtick after a backslash opens no code. A run of
+// backticks that opens no inline code stays as text whole, as in Markdown.
+func telegramRewriteLineProse(out *strings.Builder, line string, rewrite func(string) string) {
+	prose := 0
+	for index := 0; index < len(line); {
+		switch line[index] {
+		case '\\':
+			if index+1 < len(line) && strings.IndexByte(markdownPunctuation, line[index+1]) >= 0 {
+				index += 2
+				continue
+			}
+		case '`':
+			end := index + markdownBackticks(line[index:])
+			span, found := markdownCodeSpan(line, index)
 			if found {
-				closing = span.end - 1
+				end = span.end
 			}
 
-			continue
-		case !strings.HasPrefix(line[closing:], mark) || telegramIsSpace(line[closing-1]):
-			continue
-		}
+			out.WriteString(rewrite(line[prose:index]))
+			out.WriteString(line[index:end])
 
-		end := closing + len(mark)
-		if len(mark) == 1 && (line[closing-1] == char || end < len(line) && line[end] == char) {
-			continue
-		}
+			index = end
+			prose = end
 
-		if char == '_' && telegramWordChar(line[end:]) {
 			continue
 		}
 
-		return TelegramMarkdownSpan{
-			kind:    kind,
-			content: line[after:closing],
-			end:     end,
-		}, true
+		index++
 	}
 
-	return TelegramMarkdownSpan{}, false
-}
-
-// telegramBackticks returns the length of the run of backticks at the start of
-// text.
-func telegramBackticks(text string) int {
-	return len(text) - len(strings.TrimLeft(text, "`"))
-}
-
-// telegramIsSpace tells if char is a space, a tab, or a line break.
-func telegramIsSpace(char byte) bool {
-	return char == ' ' || char == '\t' || char == '\n' || char == '\r'
-}
-
-// telegramWordCharBefore tells if the character that ends text is a letter or a
-// digit.
-func telegramWordCharBefore(text string) bool {
-	char, _ := utf8.DecodeLastRuneInString(text)
-	return text != "" && (unicode.IsLetter(char) || unicode.IsDigit(char))
-}
-
-// telegramWordChar tells if the character that starts text is a letter or a
-// digit. An underscore next to one is a part of a word, such as in snake_case.
-func telegramWordChar(text string) bool {
-	char, _ := utf8.DecodeRuneInString(text)
-	return text != "" && (unicode.IsLetter(char) || unicode.IsDigit(char))
+	out.WriteString(rewrite(line[prose:]))
 }