X-Git-Url: http://git.giacomolli.uk/onechat/blobdiff_plain/de170709599e849c6b409d247b800dc68b0c7ab7..refs/heads/master:/telegram_markdown.go?ds=sidebyside diff --git a/telegram_markdown.go b/telegram_markdown.go index 16b83f2..a978036 100644 --- a/telegram_markdown.go +++ b/telegram_markdown.go @@ -3,13 +3,16 @@ package main import ( "slices" "strings" - "unicode" - "unicode/utf8" ) -// telegramMarkdownPunctuation holds the ASCII characters that a backslash makes -// plain. -const telegramMarkdownPunctuation = "!\"#$%&'()*+,-./:;<=>?@[\\]^_`{|}~" +var ( + // telegramTagEscaper writes the characters that rich Markdown reads as HTML + // as HTML entities. + telegramTagEscaper = strings.NewReplacer("&", "&", "<", "<") + + // telegramTagUnescaper reverses telegramTagEscaper. + telegramTagUnescaper = strings.NewReplacer("&", "&", "<", "<") +) // telegramMarkdownEntities converts the Markdown of a reply to plain text and // Telegram message entities. It knows bold (** and __), italic (* and _), @@ -22,6 +25,7 @@ const telegramMarkdownPunctuation = "!\"#$%&'()*+,-./:;<=>?@[\\]^_`{|}~" // change the rest of the reply. func telegramMarkdownEntities(markdown string) (string, []TelegramEntity) { var out TelegramMarkdownText + lines := strings.Split(markdown, "\n") for index := 0; index < len(lines); index++ { if index > 0 { @@ -29,9 +33,9 @@ func telegramMarkdownEntities(markdown string) (string, []TelegramEntity) { } line := lines[index] - language, fence := telegramCodeFence(line) - end := telegramClosingFence(lines, index, fence) - heading := telegramHeadingText(line) + language, fence := markdownCodeFence(line) + end := markdownClosingFence(lines, index, fence) + heading := markdownHeadingText(line) switch { case end > index: start := out.units @@ -68,60 +72,21 @@ func telegramMarkdownEntities(markdown string) (string, []TelegramEntity) { return out.text.String(), out.entities } -// telegramEscapeTags puts a backslash before each < of markdown outside inline -// code and fenced code blocks, so that rich Markdown shows a tag name as text -// and reads no raw HTML. A < that a backslash already makes plain stays as it -// is. A code block that does not close runs to the end, as in Markdown. +// telegramEscapeTags writes each & of markdown outside inline code and fenced +// code blocks as & and each < as <. Rich Markdown reads raw HTML and +// HTML entities in prose, and it does not read a backslash before < as an +// escape. With the entities, it shows a tag name and an entity that the model +// wrote in prose as text. func telegramEscapeTags(markdown string) string { - var out strings.Builder - lines := strings.Split(markdown, "\n") - for index := 0; index < len(lines); index++ { - if index > 0 { - out.WriteString("\n") - } - - line := lines[index] - _, fence := telegramCodeFence(line) - if fence == "" { - telegramEscapeLineTags(&out, line) - continue - } - - end := telegramClosingFence(lines, index, fence) - if end < 0 { - end = len(lines) - 1 - } - - out.WriteString(strings.Join(lines[index:end+1], "\n")) - index = end - } - - return out.String() -} - -// telegramCodeFence returns the language of a line that opens a fenced code -// block, and the fence. The fence is "" when line does not open a block. -func telegramCodeFence(line string) (string, string) { - trimmed := strings.TrimLeft(line, " ") - if len(line)-len(trimmed) > 3 || !strings.HasPrefix(trimmed, "```") { - return "", "" - } - - run := len(trimmed) - len(strings.TrimLeft(trimmed, "`")) - info := strings.TrimSpace(trimmed[run:]) - if strings.Contains(info, "`") { - return "", "" - } - - language, _, _ := strings.Cut(info, " ") - - return language, trimmed[:run] + return telegramRewriteProse(markdown, telegramTagEscaper.Replace) } -// telegramClosesFence tells if line closes the code block of fence. -func telegramClosesFence(line, fence string) bool { - trimmed := strings.TrimSpace(line) - return strings.HasPrefix(trimmed, fence) && strings.Trim(trimmed, "`") == "" +// telegramUnescapeTags gives back the Markdown that telegramEscapeTags escaped. +// It is exact, because after the escape each & outside code starts & or +// <. It also works on a part of telegramSplitRich, because a part has the +// same code as the text that it comes from. +func telegramUnescapeTags(markdown string) string { + return telegramRewriteProse(markdown, telegramTagUnescaper.Replace) } // telegramUTF16Units returns the number of UTF-16 code units of text. @@ -149,6 +114,7 @@ type TelegramMarkdownText struct { // add appends text. func (m *TelegramMarkdownText) add(text string) { m.text.WriteString(text) + m.units += telegramUTF16Units(text) } @@ -167,21 +133,23 @@ func (m *TelegramMarkdownText) inline(line string) { plain := 0 index := 0 for index < len(line) { - if line[index] == '\\' && index+1 < len(line) && strings.IndexByte(telegramMarkdownPunctuation, line[index+1]) >= 0 { + if line[index] == '\\' && index+1 < len(line) && strings.IndexByte(markdownPunctuation, line[index+1]) >= 0 { m.add(line[plain:index]) + plain = index + 1 index += 2 continue } - span, found := telegramMarkSpan(line, index) + span, found := markdownMarkSpan(line, index) if !found { index++ continue } m.add(line[plain:index]) + start := m.units if span.kind == "code" { m.add(span.content) @@ -203,269 +171,69 @@ func (m *TelegramMarkdownText) inline(line string) { m.add(line[plain:]) } -// TelegramMarkdownSpan is an inline mark of a line: the entity type, the -// content inside the marks, the index after the closing mark, and the URL of a -// link. -type TelegramMarkdownSpan struct { - kind string - content string - end int - url string -} - -// telegramClosingFence returns the index of the line after index that closes -// the code block of fence, or -1 when fence is "" or no line closes it. -func telegramClosingFence(lines []string, index int, fence string) int { - if fence == "" { - return -1 - } +// telegramRewriteProse returns markdown with rewrite applied to each part of a +// line outside inline code and fenced code blocks. A code block that does not +// close runs to the end, as in Markdown. +func telegramRewriteProse(markdown string, rewrite func(string) string) string { + var out strings.Builder - for end := index + 1; end < len(lines); end++ { - if telegramClosesFence(lines[end], fence) { - return end + lines := strings.Split(markdown, "\n") + for index := 0; index < len(lines); index++ { + if index > 0 { + out.WriteString("\n") } - } - return -1 -} + line := lines[index] + _, fence := markdownCodeFence(line) + if fence == "" { + telegramRewriteLineProse(&out, line, rewrite) -// telegramHeadingText returns the text of an ATX heading line without its -// marks, or "" when line is not a heading or has no text. -func telegramHeadingText(line string) string { - trimmed := strings.TrimLeft(line, " ") - if len(line)-len(trimmed) > 3 { - return "" - } + continue + } - level := len(trimmed) - len(strings.TrimLeft(trimmed, "#")) - if level == 0 || level > 6 { - return "" - } + end := markdownClosingFence(lines, index, fence) + if end < 0 { + end = len(lines) - 1 + } - rest := trimmed[level:] - if rest != "" && rest[0] != ' ' && rest[0] != '\t' { - return "" - } + out.WriteString(strings.Join(lines[index:end+1], "\n")) - rest = strings.TrimSpace(rest) - closed := strings.TrimRight(rest, "#") - if closed == "" || strings.HasSuffix(closed, " ") { - rest = strings.TrimSpace(closed) + index = end } - return rest + return out.String() } -// telegramEscapeLineTags writes line to out with a backslash before each < that -// is outside inline code and not already plain. A run of backticks that opens -// no inline code stays as text whole, as in Markdown. -func telegramEscapeLineTags(out *strings.Builder, line string) { +// telegramRewriteLineProse writes line to out with rewrite applied to each part +// outside inline code. A backtick after a backslash opens no code. A run of +// backticks that opens no inline code stays as text whole, as in Markdown. +func telegramRewriteLineProse(out *strings.Builder, line string, rewrite func(string) string) { + prose := 0 for index := 0; index < len(line); { switch line[index] { case '\\': - if index+1 < len(line) && strings.IndexByte(telegramMarkdownPunctuation, line[index+1]) >= 0 { - out.WriteString(line[index : index+2]) + if index+1 < len(line) && strings.IndexByte(markdownPunctuation, line[index+1]) >= 0 { index += 2 - continue } case '`': - end := index + telegramBackticks(line[index:]) - span, found := telegramCodeSpan(line, index) + end := index + markdownBackticks(line[index:]) + span, found := markdownCodeSpan(line, index) if found { end = span.end } + out.WriteString(rewrite(line[prose:index])) out.WriteString(line[index:end]) + index = end + prose = end continue - case '<': - out.WriteByte('\\') } - out.WriteByte(line[index]) index++ } -} - -// telegramMarkSpan returns the inline mark that starts at index of line and -// closes in line. It returns false when no mark starts there or the mark does -// not close. -func telegramMarkSpan(line string, index int) (TelegramMarkdownSpan, bool) { - switch line[index] { - case '`': - return telegramCodeSpan(line, index) - case '[': - return telegramLinkSpan(line, index) - case '~': - return telegramDelimited(line, index, "~~", "strikethrough") - case '*', '_': - double := line[index : index+1] - double += double - span, found := telegramDelimited(line, index, double, "bold") - if found { - return span, true - } - - return telegramDelimited(line, index, line[index:index+1], "italic") - default: - return TelegramMarkdownSpan{}, false - } -} - -// telegramCodeSpan returns the inline code that starts at index: a run of -// backticks, the code, and a run of the same length. One space on each side of -// the code is left out, as in Markdown. -func telegramCodeSpan(line string, index int) (TelegramMarkdownSpan, bool) { - run := telegramBackticks(line[index:]) - for search := index + run; search < len(line); { - next := strings.IndexByte(line[search:], '`') - if next < 0 { - break - } - - next += search - length := telegramBackticks(line[next:]) - if length != run { - search = next + length - continue - } - - content := line[index+run : next] - if strings.TrimSpace(content) == "" { - break - } - - if len(content) > 1 && content[0] == ' ' && content[len(content)-1] == ' ' { - content = content[1 : len(content)-1] - } - - return TelegramMarkdownSpan{ - kind: "code", - content: content, - end: next + run, - }, true - } - - return TelegramMarkdownSpan{}, false -} - -// telegramLinkSpan returns the link [text](url) that starts at index. The text -// can hold brackets in pairs, and the URL holds no space. -func telegramLinkSpan(line string, index int) (TelegramMarkdownSpan, bool) { - depth := 0 - for closing := index; closing < len(line); closing++ { - switch line[closing] { - case '\\': - closing++ - case '[': - depth++ - case ']': - depth-- - } - - if depth > 0 { - continue - } - - rest := line[closing+1:] - if closing == index+1 || !strings.HasPrefix(rest, "(") { - return TelegramMarkdownSpan{}, false - } - - url, _, found := strings.Cut(rest[1:], ")") - if !found || url == "" || strings.ContainsAny(url, " \t") { - return TelegramMarkdownSpan{}, false - } - - return TelegramMarkdownSpan{ - kind: "text_link", - content: line[index+1 : closing], - end: closing + 2 + len(url) + 1, - url: url, - }, true - } - - return TelegramMarkdownSpan{}, false -} - -// telegramDelimited returns the span of kind between the mark at index and the -// same mark later in line. An opening mark is followed by a character that is -// not a space, and a closing mark follows one. A single mark is not a part of a -// double one. A mark of underscores does not open or close inside a word, so -// snake_case stays as text. A code span inside the content is skipped whole. -func telegramDelimited(line string, index int, mark, kind string) (TelegramMarkdownSpan, bool) { - char := mark[0] - after := index + len(mark) - if !strings.HasPrefix(line[index:], mark) || after >= len(line) || telegramIsSpace(line[after]) { - return TelegramMarkdownSpan{}, false - } - - if len(mark) == 1 && (line[after] == char || index > 0 && line[index-1] == char) { - return TelegramMarkdownSpan{}, false - } - - if char == '_' && index > 0 && telegramWordCharBefore(line[:index]) { - return TelegramMarkdownSpan{}, false - } - - for closing := after + 1; closing < len(line); closing++ { - switch { - case line[closing] == '\\': - closing++ - continue - case line[closing] == '`': - span, found := telegramCodeSpan(line, closing) - if found { - closing = span.end - 1 - } - - continue - case !strings.HasPrefix(line[closing:], mark) || telegramIsSpace(line[closing-1]): - continue - } - - end := closing + len(mark) - if len(mark) == 1 && (line[closing-1] == char || end < len(line) && line[end] == char) { - continue - } - - if char == '_' && telegramWordChar(line[end:]) { - continue - } - - return TelegramMarkdownSpan{ - kind: kind, - content: line[after:closing], - end: end, - }, true - } - - return TelegramMarkdownSpan{}, false -} - -// telegramBackticks returns the length of the run of backticks at the start of -// text. -func telegramBackticks(text string) int { - return len(text) - len(strings.TrimLeft(text, "`")) -} - -// telegramIsSpace tells if char is a space, a tab, or a line break. -func telegramIsSpace(char byte) bool { - return char == ' ' || char == '\t' || char == '\n' || char == '\r' -} - -// telegramWordCharBefore tells if the character that ends text is a letter or a -// digit. -func telegramWordCharBefore(text string) bool { - char, _ := utf8.DecodeLastRuneInString(text) - return text != "" && (unicode.IsLetter(char) || unicode.IsDigit(char)) -} -// telegramWordChar tells if the character that starts text is a letter or a -// digit. An underscore next to one is a part of a word, such as in snake_case. -func telegramWordChar(text string) bool { - char, _ := utf8.DecodeRuneInString(text) - return text != "" && (unicode.IsLetter(char) || unicode.IsDigit(char)) + out.WriteString(rewrite(line[prose:])) }