Version the package and command from the commit

[onechat] / telegram_markdown.go

package main

import (
        "slices"
        "strings"
        "unicode"
        "unicode/utf8"
)

// telegramMarkdownPunctuation holds the ASCII characters that a backslash makes
// plain.
const telegramMarkdownPunctuation = "!\"#$%&'()*+,-./:;<=>?@[\\]^_`{|}~"

var (
        // telegramTagEscaper writes the characters that rich Markdown reads as HTML
        // as HTML entities.
        telegramTagEscaper = strings.NewReplacer("&", "&amp;", "<", "&lt;")

        // telegramTagUnescaper reverses telegramTagEscaper.
        telegramTagUnescaper = strings.NewReplacer("&amp;", "&", "&lt;", "<")
)

// telegramMarkdownEntities converts the Markdown of a reply to plain text and
// Telegram message entities. It knows bold (** and __), italic (* and _),
// strikethrough (~~), inline code, fenced code blocks with an optional
// language, links [text](url), and ATX headings, which become bold. Lists,
// quotes, and tables stay as their text. A mark that does not close stays as
// text with no entity, and code holds no entity.
//
// Inline marks do not cross a line break, so a mark that does not close cannot
// change the rest of the reply.
func telegramMarkdownEntities(markdown string) (string, []TelegramEntity) {
        var out TelegramMarkdownText

        lines := strings.Split(markdown, "\n")
        for index := 0; index < len(lines); index++ {
                if index > 0 {
                        out.add("\n")
                }

                line := lines[index]
                language, fence := telegramCodeFence(line)
                end := telegramClosingFence(lines, index, fence)
                heading := telegramHeadingText(line)
                switch {
                case end > index:
                        start := out.units
                        out.add(strings.Join(lines[index+1:end], "\n"))
                        out.entity(TelegramEntity{
                                Type:     "pre",
                                Offset:   start,
                                Length:   out.units - start,
                                Language: language,
                        })

                        index = end
                case heading != "":
                        start := out.units
                        out.inline(heading)
                        out.entity(TelegramEntity{
                                Type:   "bold",
                                Offset: start,
                                Length: out.units - start,
                        })
                default:
                        out.inline(line)
                }
        }

        slices.SortStableFunc(out.entities, func(a, b TelegramEntity) int {
                if a.Offset != b.Offset {
                        return a.Offset - b.Offset
                }

                return b.Length - a.Length
        })

        return out.text.String(), out.entities
}

// telegramEscapeTags writes each & of markdown outside inline code and fenced
// code blocks as &amp; and each < as &lt;. Rich Markdown reads raw HTML and
// HTML entities in prose, and it does not read a backslash before < as an
// escape. With the entities, it shows a tag name and an entity that the model
// wrote in prose as text.
func telegramEscapeTags(markdown string) string {
        return telegramRewriteProse(markdown, telegramTagEscaper.Replace)
}

// telegramUnescapeTags gives back the Markdown that telegramEscapeTags escaped.
// It is exact, because after the escape each & outside code starts &amp; or
// &lt;. It also works on a part of telegramSplitRich, because a part has the
// same code as the text that it comes from.
func telegramUnescapeTags(markdown string) string {
        return telegramRewriteProse(markdown, telegramTagUnescaper.Replace)
}

// telegramCodeFence returns the language of a line that opens a fenced code
// block, and the fence. The fence is "" when line does not open a block.
func telegramCodeFence(line string) (string, string) {
        trimmed := strings.TrimLeft(line, " ")
        if len(line)-len(trimmed) > 3 || !strings.HasPrefix(trimmed, "```") {
                return "", ""
        }

        run := len(trimmed) - len(strings.TrimLeft(trimmed, "`"))
        info := strings.TrimSpace(trimmed[run:])
        if strings.Contains(info, "`") {
                return "", ""
        }

        language, _, _ := strings.Cut(info, " ")

        return language, trimmed[:run]
}

// telegramClosesFence tells if line closes the code block of fence.
func telegramClosesFence(line, fence string) bool {
        trimmed := strings.TrimSpace(line)
        return strings.HasPrefix(trimmed, fence) && strings.Trim(trimmed, "`") == ""
}

// telegramUTF16Units returns the number of UTF-16 code units of text.
func telegramUTF16Units(text string) int {
        units := 0
        for _, char := range text {
                units++
                if char > 0xFFFF {
                        units++
                }
        }

        return units
}

// TelegramMarkdownText is the plain text of a Markdown reply and its Telegram
// entities, while the conversion builds it. Units counts the UTF-16 code units
// of text, the unit of the entity offsets and lengths.
type TelegramMarkdownText struct {
        text     strings.Builder
        units    int
        entities []TelegramEntity
}

// add appends text.
func (m *TelegramMarkdownText) add(text string) {
        m.text.WriteString(text)

        m.units += telegramUTF16Units(text)
}

// entity keeps entity, unless it is empty, because Telegram refuses an entity
// of length 0.
func (m *TelegramMarkdownText) entity(entity TelegramEntity) {
        if entity.Length > 0 {
                m.entities = append(m.entities, entity)
        }
}

// inline appends one line of Markdown with its inline marks as entities. A
// backslash before an ASCII punctuation character makes the character plain
// text.
func (m *TelegramMarkdownText) inline(line string) {
        plain := 0
        index := 0
        for index < len(line) {
                if line[index] == '\\' && index+1 < len(line) && strings.IndexByte(telegramMarkdownPunctuation, line[index+1]) >= 0 {
                        m.add(line[plain:index])

                        plain = index + 1
                        index += 2

                        continue
                }

                span, found := telegramMarkSpan(line, index)
                if !found {
                        index++
                        continue
                }

                m.add(line[plain:index])

                start := m.units
                if span.kind == "code" {
                        m.add(span.content)
                } else {
                        m.inline(span.content)
                }

                m.entity(TelegramEntity{
                        Type:   span.kind,
                        Offset: start,
                        Length: m.units - start,
                        URL:    span.url,
                })

                index = span.end
                plain = span.end
        }

        m.add(line[plain:])
}

// TelegramMarkdownSpan is an inline mark of a line: the entity type, the
// content inside the marks, the index after the closing mark, and the URL of a
// link.
type TelegramMarkdownSpan struct {
        kind    string
        content string
        end     int
        url     string
}

// telegramClosingFence returns the index of the line after index that closes
// the code block of fence, or -1 when fence is "" or no line closes it.
func telegramClosingFence(lines []string, index int, fence string) int {
        if fence == "" {
                return -1
        }

        for end := index + 1; end < len(lines); end++ {
                if telegramClosesFence(lines[end], fence) {
                        return end
                }
        }

        return -1
}

// telegramHeadingText returns the text of an ATX heading line without its
// marks, or "" when line is not a heading or has no text.
func telegramHeadingText(line string) string {
        trimmed := strings.TrimLeft(line, " ")
        if len(line)-len(trimmed) > 3 {
                return ""
        }

        level := len(trimmed) - len(strings.TrimLeft(trimmed, "#"))
        if level == 0 || level > 6 {
                return ""
        }

        rest := trimmed[level:]
        if rest != "" && rest[0] != ' ' && rest[0] != '\t' {
                return ""
        }

        rest = strings.TrimSpace(rest)
        closed := strings.TrimRight(rest, "#")
        if closed == "" || strings.HasSuffix(closed, " ") {
                rest = strings.TrimSpace(closed)
        }

        return rest
}

// telegramRewriteProse returns markdown with rewrite applied to each part of a
// line outside inline code and fenced code blocks. A code block that does not
// close runs to the end, as in Markdown.
func telegramRewriteProse(markdown string, rewrite func(string) string) string {
        var out strings.Builder

        lines := strings.Split(markdown, "\n")
        for index := 0; index < len(lines); index++ {
                if index > 0 {
                        out.WriteString("\n")
                }

                line := lines[index]
                _, fence := telegramCodeFence(line)
                if fence == "" {
                        telegramRewriteLineProse(&out, line, rewrite)

                        continue
                }

                end := telegramClosingFence(lines, index, fence)
                if end < 0 {
                        end = len(lines) - 1
                }

                out.WriteString(strings.Join(lines[index:end+1], "\n"))

                index = end
        }

        return out.String()
}

// telegramRewriteLineProse writes line to out with rewrite applied to each part
// outside inline code. A backtick after a backslash opens no code. A run of
// backticks that opens no inline code stays as text whole, as in Markdown.
func telegramRewriteLineProse(out *strings.Builder, line string, rewrite func(string) string) {
        prose := 0
        for index := 0; index < len(line); {
                switch line[index] {
                case '\\':
                        if index+1 < len(line) && strings.IndexByte(telegramMarkdownPunctuation, line[index+1]) >= 0 {
                                index += 2
                                continue
                        }
                case '`':
                        end := index + telegramBackticks(line[index:])
                        span, found := telegramCodeSpan(line, index)
                        if found {
                                end = span.end
                        }

                        out.WriteString(rewrite(line[prose:index]))
                        out.WriteString(line[index:end])

                        index = end
                        prose = end

                        continue
                }

                index++
        }

        out.WriteString(rewrite(line[prose:]))
}

// telegramMarkSpan returns the inline mark that starts at index of line and
// closes in line. It returns false when no mark starts there or the mark does
// not close.
func telegramMarkSpan(line string, index int) (TelegramMarkdownSpan, bool) {
        switch line[index] {
        case '`':
                return telegramCodeSpan(line, index)
        case '[':
                return telegramLinkSpan(line, index)
        case '~':
                return telegramDelimited(line, index, "~~", "strikethrough")
        case '*', '_':
                double := line[index : index+1]
                double += double
                span, found := telegramDelimited(line, index, double, "bold")
                if found {
                        return span, true
                }

                return telegramDelimited(line, index, line[index:index+1], "italic")
        default:
                return TelegramMarkdownSpan{}, false
        }
}

// telegramCodeSpan returns the inline code that starts at index: a run of
// backticks, the code, and a run of the same length. One space on each side of
// the code is left out, as in Markdown.
func telegramCodeSpan(line string, index int) (TelegramMarkdownSpan, bool) {
        run := telegramBackticks(line[index:])
        for search := index + run; search < len(line); {
                next := strings.IndexByte(line[search:], '`')
                if next < 0 {
                        break
                }

                next += search
                length := telegramBackticks(line[next:])
                if length != run {
                        search = next + length
                        continue
                }

                content := line[index+run : next]
                if strings.TrimSpace(content) == "" {
                        break
                }

                if len(content) > 1 && content[0] == ' ' && content[len(content)-1] == ' ' {
                        content = content[1 : len(content)-1]
                }

                return TelegramMarkdownSpan{
                        kind:    "code",
                        content: content,
                        end:     next + run,
                }, true
        }

        return TelegramMarkdownSpan{}, false
}

// telegramLinkSpan returns the link [text](url) that starts at index. The text
// can hold brackets in pairs, and the URL holds no space.
func telegramLinkSpan(line string, index int) (TelegramMarkdownSpan, bool) {
        depth := 0
        for closing := index; closing < len(line); closing++ {
                switch line[closing] {
                case '\\':
                        closing++
                case '[':
                        depth++
                case ']':
                        depth--
                }

                if depth > 0 {
                        continue
                }

                rest := line[closing+1:]
                if closing == index+1 || !strings.HasPrefix(rest, "(") {
                        return TelegramMarkdownSpan{}, false
                }

                url, _, found := strings.Cut(rest[1:], ")")
                if !found || url == "" || strings.ContainsAny(url, " \t") {
                        return TelegramMarkdownSpan{}, false
                }

                return TelegramMarkdownSpan{
                        kind:    "text_link",
                        content: line[index+1 : closing],
                        end:     closing + 2 + len(url) + 1,
                        url:     url,
                }, true
        }

        return TelegramMarkdownSpan{}, false
}

// telegramDelimited returns the span of kind between the mark at index and the
// same mark later in line. An opening mark is followed by a character that is
// not a space, and a closing mark follows one. A single mark is not a part of a
// double one. A mark of underscores does not open or close inside a word, so
// snake_case stays as text. A code span inside the content is skipped whole.
func telegramDelimited(line string, index int, mark, kind string) (TelegramMarkdownSpan, bool) {
        char := mark[0]
        after := index + len(mark)
        if !strings.HasPrefix(line[index:], mark) || after >= len(line) || telegramIsSpace(line[after]) {
                return TelegramMarkdownSpan{}, false
        }

        if len(mark) == 1 && (line[after] == char || index > 0 && line[index-1] == char) {
                return TelegramMarkdownSpan{}, false
        }

        if char == '_' && index > 0 && telegramWordCharBefore(line[:index]) {
                return TelegramMarkdownSpan{}, false
        }

        for closing := after + 1; closing < len(line); closing++ {
                switch {
                case line[closing] == '\\':
                        closing++
                        continue
                case line[closing] == '`':
                        span, found := telegramCodeSpan(line, closing)
                        if found {
                                closing = span.end - 1
                        }

                        continue
                case !strings.HasPrefix(line[closing:], mark) || telegramIsSpace(line[closing-1]):
                        continue
                }

                end := closing + len(mark)
                if len(mark) == 1 && (line[closing-1] == char || end < len(line) && line[end] == char) {
                        continue
                }

                if char == '_' && telegramWordChar(line[end:]) {
                        continue
                }

                return TelegramMarkdownSpan{
                        kind:    kind,
                        content: line[after:closing],
                        end:     end,
                }, true
        }

        return TelegramMarkdownSpan{}, false
}

// telegramBackticks returns the length of the run of backticks at the start of
// text.
func telegramBackticks(text string) int {
        return len(text) - len(strings.TrimLeft(text, "`"))
}

// telegramIsSpace tells if char is a space, a tab, or a line break.
func telegramIsSpace(char byte) bool {
        return char == ' ' || char == '\t' || char == '\n' || char == '\r'
}

// telegramWordCharBefore tells if the character that ends text is a letter or a
// digit.
func telegramWordCharBefore(text string) bool {
        char, _ := utf8.DecodeLastRuneInString(text)
        return text != "" && (unicode.IsLetter(char) || unicode.IsDigit(char))
}

// telegramWordChar tells if the character that starts text is a letter or a
// digit. An underscore next to one is a part of a word, such as in snake_case.
func telegramWordChar(text string) bool {
        char, _ := utf8.DecodeRuneInString(text)
        return text != "" && (unicode.IsLetter(char) || unicode.IsDigit(char))
}