Reflow comments over the reflow width

[onechat] / telegram_markdown.go

diff --git a/telegram_markdown.go b/telegram_markdown.go
index 16b83f2f851d81bddd19353a1af8e6c43dfb6c97..fc686344556bec329c0c1c7d5bbcb674133fdafd 100644 (file)
--- a/telegram_markdown.go
+++ b/telegram_markdown.go
@@ -3,13 +3,16 @@ package main
 import (
        "slices"
        "strings"
 import (
        "slices"
        "strings"

-       "unicode"
-       "unicode/utf8"

 )
 
 )
 

-// telegramMarkdownPunctuation holds the ASCII characters that a backslash makes
-// plain.
-const telegramMarkdownPunctuation = "!\"#$%&'()*+,-./:;<=>?@[\\]^_`{|}~"
+var (
+       // telegramTagEscaper writes the characters that rich Markdown reads as
+       // HTML as HTML entities.
+       telegramTagEscaper = strings.NewReplacer("&", "&amp;", "<", "&lt;")
+
+       // telegramTagUnescaper reverses telegramTagEscaper.
+       telegramTagUnescaper = strings.NewReplacer("&amp;", "&", "&lt;", "<")
+)

 
 // telegramMarkdownEntities converts the Markdown of a reply to plain text and
 // Telegram message entities. It knows bold (** and __), italic (* and _),
 
 // telegramMarkdownEntities converts the Markdown of a reply to plain text and
 // Telegram message entities. It knows bold (** and __), italic (* and _),


@@ -22,6 +25,7 @@ const telegramMarkdownPunctuation = "!\"#$%&'()*+,-./:;<=>?@[\\]^_`{|}~"
 // change the rest of the reply.
 func telegramMarkdownEntities(markdown string) (string, []TelegramEntity) {
        var out TelegramMarkdownText
 // change the rest of the reply.
 func telegramMarkdownEntities(markdown string) (string, []TelegramEntity) {
        var out TelegramMarkdownText

+

        lines := strings.Split(markdown, "\n")
        for index := 0; index < len(lines); index++ {
                if index > 0 {
        lines := strings.Split(markdown, "\n")
        for index := 0; index < len(lines); index++ {
                if index > 0 {


@@ -29,9 +33,9 @@ func telegramMarkdownEntities(markdown string) (string, []TelegramEntity) {
                }
 
                line := lines[index]
                }
 
                line := lines[index]

-               language, fence := telegramCodeFence(line)
-               end := telegramClosingFence(lines, index, fence)
-               heading := telegramHeadingText(line)
+               language, fence := markdownCodeFence(line)
+               end := markdownClosingFence(lines, index, fence)
+               heading := markdownHeadingText(line)

                switch {
                case end > index:
                        start := out.units
                switch {
                case end > index:
                        start := out.units


@@ -68,60 +72,21 @@ func telegramMarkdownEntities(markdown string) (string, []TelegramEntity) {
        return out.text.String(), out.entities
 }
 
        return out.text.String(), out.entities
 }
 

-// telegramEscapeTags puts a backslash before each < of markdown outside inline
-// code and fenced code blocks, so that rich Markdown shows a tag name as text
-// and reads no raw HTML. A < that a backslash already makes plain stays as it
-// is. A code block that does not close runs to the end, as in Markdown.
+// telegramEscapeTags writes each & of markdown outside inline code and fenced
+// code blocks as &amp; and each < as &lt;. Rich Markdown reads raw HTML and
+// HTML entities in prose, and it does not read a backslash before < as an
+// escape. With the entities, it shows a tag name and an entity that the model
+// wrote in prose as text.

 func telegramEscapeTags(markdown string) string {
 func telegramEscapeTags(markdown string) string {

-       var out strings.Builder
-       lines := strings.Split(markdown, "\n")
-       for index := 0; index < len(lines); index++ {
-               if index > 0 {
-                       out.WriteString("\n")
-               }
-
-               line := lines[index]
-               _, fence := telegramCodeFence(line)
-               if fence == "" {
-                       telegramEscapeLineTags(&out, line)
-                       continue
-               }
-
-               end := telegramClosingFence(lines, index, fence)
-               if end < 0 {
-                       end = len(lines) - 1
-               }
-
-               out.WriteString(strings.Join(lines[index:end+1], "\n"))
-               index = end
-       }
-
-       return out.String()
-}
-
-// telegramCodeFence returns the language of a line that opens a fenced code
-// block, and the fence. The fence is "" when line does not open a block.
-func telegramCodeFence(line string) (string, string) {
-       trimmed := strings.TrimLeft(line, " ")
-       if len(line)-len(trimmed) > 3 || !strings.HasPrefix(trimmed, "```") {
-               return "", ""
-       }
-
-       run := len(trimmed) - len(strings.TrimLeft(trimmed, "`"))
-       info := strings.TrimSpace(trimmed[run:])
-       if strings.Contains(info, "`") {
-               return "", ""
-       }
-
-       language, _, _ := strings.Cut(info, " ")
-
-       return language, trimmed[:run]
+       return telegramRewriteProse(markdown, telegramTagEscaper.Replace)

 }
 
 }
 

-// telegramClosesFence tells if line closes the code block of fence.
-func telegramClosesFence(line, fence string) bool {
-       trimmed := strings.TrimSpace(line)
-       return strings.HasPrefix(trimmed, fence) && strings.Trim(trimmed, "`") == ""
+// telegramUnescapeTags gives back the Markdown that telegramEscapeTags escaped.
+// It is exact, because after the escape each & outside code starts &amp; or
+// &lt;. It also works on a part of telegramSplitRich, because a part has the
+// same code as the text that it comes from.
+func telegramUnescapeTags(markdown string) string {
+       return telegramRewriteProse(markdown, telegramTagUnescaper.Replace)

 }
 
 // telegramUTF16Units returns the number of UTF-16 code units of text.
 }
 
 // telegramUTF16Units returns the number of UTF-16 code units of text.


@@ -149,6 +114,7 @@ type TelegramMarkdownText struct {
 // add appends text.
 func (m *TelegramMarkdownText) add(text string) {
        m.text.WriteString(text)
 // add appends text.
 func (m *TelegramMarkdownText) add(text string) {
        m.text.WriteString(text)

+

        m.units += telegramUTF16Units(text)
 }
 
        m.units += telegramUTF16Units(text)
 }
 


@@ -167,21 +133,23 @@ func (m *TelegramMarkdownText) inline(line string) {
        plain := 0
        index := 0
        for index < len(line) {
        plain := 0
        index := 0
        for index < len(line) {

-               if line[index] == '\\' && index+1 < len(line) && strings.IndexByte(telegramMarkdownPunctuation, line[index+1]) >= 0 {
+               if line[index] == '\\' && index+1 < len(line) && strings.IndexByte(markdownPunctuation, line[index+1]) >= 0 {

                        m.add(line[plain:index])
                        m.add(line[plain:index])

+

                        plain = index + 1
                        index += 2
 
                        continue
                }
 
                        plain = index + 1
                        index += 2
 
                        continue
                }
 

-               span, found := telegramMarkSpan(line, index)
+               span, found := markdownMarkSpan(line, index)

                if !found {
                        index++
                        continue
                }
 
                m.add(line[plain:index])
                if !found {
                        index++
                        continue
                }
 
                m.add(line[plain:index])

+

                start := m.units
                if span.kind == "code" {
                        m.add(span.content)
                start := m.units
                if span.kind == "code" {
                        m.add(span.content)


@@ -203,269 +171,69 @@ func (m *TelegramMarkdownText) inline(line string) {
        m.add(line[plain:])
 }
 
        m.add(line[plain:])
 }
 

-// TelegramMarkdownSpan is an inline mark of a line: the entity type, the
-// content inside the marks, the index after the closing mark, and the URL of a
-// link.
-type TelegramMarkdownSpan struct {
-       kind    string
-       content string
-       end     int
-       url     string
-}
-
-// telegramClosingFence returns the index of the line after index that closes
-// the code block of fence, or -1 when fence is "" or no line closes it.
-func telegramClosingFence(lines []string, index int, fence string) int {
-       if fence == "" {
-               return -1
-       }
+// telegramRewriteProse returns markdown with rewrite applied to each part of a
+// line outside inline code and fenced code blocks. A code block that does not
+// close runs to the end, as in Markdown.
+func telegramRewriteProse(markdown string, rewrite func(string) string) string {
+       var out strings.Builder

 
 

-       for end := index + 1; end < len(lines); end++ {
-               if telegramClosesFence(lines[end], fence) {
-                       return end
+       lines := strings.Split(markdown, "\n")
+       for index := 0; index < len(lines); index++ {
+               if index > 0 {
+                       out.WriteString("\n")

                }
                }

-       }

 
 

-       return -1
-}
+               line := lines[index]
+               _, fence := markdownCodeFence(line)
+               if fence == "" {
+                       telegramRewriteLineProse(&out, line, rewrite)

 
 

-// telegramHeadingText returns the text of an ATX heading line without its
-// marks, or "" when line is not a heading or has no text.
-func telegramHeadingText(line string) string {
-       trimmed := strings.TrimLeft(line, " ")
-       if len(line)-len(trimmed) > 3 {
-               return ""
-       }
+                       continue
+               }

 
 

-       level := len(trimmed) - len(strings.TrimLeft(trimmed, "#"))
-       if level == 0 || level > 6 {
-               return ""
-       }
+               end := markdownClosingFence(lines, index, fence)
+               if end < 0 {
+                       end = len(lines) - 1
+               }

 
 

-       rest := trimmed[level:]
-       if rest != "" && rest[0] != ' ' && rest[0] != '\t' {
-               return ""
-       }
+               out.WriteString(strings.Join(lines[index:end+1], "\n"))

 
 

-       rest = strings.TrimSpace(rest)
-       closed := strings.TrimRight(rest, "#")
-       if closed == "" || strings.HasSuffix(closed, " ") {
-               rest = strings.TrimSpace(closed)
+               index = end

        }
 
        }
 

-       return rest
+       return out.String()

 }
 
 }
 

-// telegramEscapeLineTags writes line to out with a backslash before each < that
-// is outside inline code and not already plain. A run of backticks that opens
-// no inline code stays as text whole, as in Markdown.
-func telegramEscapeLineTags(out *strings.Builder, line string) {
+// telegramRewriteLineProse writes line to out with rewrite applied to each part
+// outside inline code. A backtick after a backslash opens no code. A run of
+// backticks that opens no inline code stays as text whole, as in Markdown.
+func telegramRewriteLineProse(out *strings.Builder, line string, rewrite func(string) string) {
+       prose := 0

        for index := 0; index < len(line); {
                switch line[index] {
                case '\\':
        for index := 0; index < len(line); {
                switch line[index] {
                case '\\':

-                       if index+1 < len(line) && strings.IndexByte(telegramMarkdownPunctuation, line[index+1]) >= 0 {
-                               out.WriteString(line[index : index+2])
+                       if index+1 < len(line) && strings.IndexByte(markdownPunctuation, line[index+1]) >= 0 {

                                index += 2
                                index += 2

-

                                continue
                        }
                case '`':
                                continue
                        }
                case '`':

-                       end := index + telegramBackticks(line[index:])
-                       span, found := telegramCodeSpan(line, index)
+                       end := index + markdownBackticks(line[index:])
+                       span, found := markdownCodeSpan(line, index)

                        if found {
                                end = span.end
                        }
 
                        if found {
                                end = span.end
                        }
 

+                       out.WriteString(rewrite(line[prose:index]))

                        out.WriteString(line[index:end])
                        out.WriteString(line[index:end])

+

                        index = end
                        index = end

+                       prose = end

 
                        continue
 
                        continue

-               case '<':
-                       out.WriteByte('\\')

                }
 
                }
 

-               out.WriteByte(line[index])

                index++
        }
                index++
        }

-}
-
-// telegramMarkSpan returns the inline mark that starts at index of line and
-// closes in line. It returns false when no mark starts there or the mark does
-// not close.
-func telegramMarkSpan(line string, index int) (TelegramMarkdownSpan, bool) {
-       switch line[index] {
-       case '`':
-               return telegramCodeSpan(line, index)
-       case '[':
-               return telegramLinkSpan(line, index)
-       case '~':
-               return telegramDelimited(line, index, "~~", "strikethrough")
-       case '*', '_':
-               double := line[index : index+1]
-               double += double
-               span, found := telegramDelimited(line, index, double, "bold")
-               if found {
-                       return span, true
-               }
-
-               return telegramDelimited(line, index, line[index:index+1], "italic")
-       default:
-               return TelegramMarkdownSpan{}, false
-       }
-}
-
-// telegramCodeSpan returns the inline code that starts at index: a run of
-// backticks, the code, and a run of the same length. One space on each side of
-// the code is left out, as in Markdown.
-func telegramCodeSpan(line string, index int) (TelegramMarkdownSpan, bool) {
-       run := telegramBackticks(line[index:])
-       for search := index + run; search < len(line); {
-               next := strings.IndexByte(line[search:], '`')
-               if next < 0 {
-                       break
-               }
-
-               next += search
-               length := telegramBackticks(line[next:])
-               if length != run {
-                       search = next + length
-                       continue
-               }
-
-               content := line[index+run : next]
-               if strings.TrimSpace(content) == "" {
-                       break
-               }
-
-               if len(content) > 1 && content[0] == ' ' && content[len(content)-1] == ' ' {
-                       content = content[1 : len(content)-1]
-               }
-
-               return TelegramMarkdownSpan{
-                       kind:    "code",
-                       content: content,
-                       end:     next + run,
-               }, true
-       }
-
-       return TelegramMarkdownSpan{}, false
-}
-
-// telegramLinkSpan returns the link [text](url) that starts at index. The text
-// can hold brackets in pairs, and the URL holds no space.
-func telegramLinkSpan(line string, index int) (TelegramMarkdownSpan, bool) {
-       depth := 0
-       for closing := index; closing < len(line); closing++ {
-               switch line[closing] {
-               case '\\':
-                       closing++
-               case '[':
-                       depth++
-               case ']':
-                       depth--
-               }
-
-               if depth > 0 {
-                       continue
-               }
-
-               rest := line[closing+1:]
-               if closing == index+1 || !strings.HasPrefix(rest, "(") {
-                       return TelegramMarkdownSpan{}, false
-               }
-
-               url, _, found := strings.Cut(rest[1:], ")")
-               if !found || url == "" || strings.ContainsAny(url, " \t") {
-                       return TelegramMarkdownSpan{}, false
-               }
-
-               return TelegramMarkdownSpan{
-                       kind:    "text_link",
-                       content: line[index+1 : closing],
-                       end:     closing + 2 + len(url) + 1,
-                       url:     url,
-               }, true
-       }
-
-       return TelegramMarkdownSpan{}, false
-}
-
-// telegramDelimited returns the span of kind between the mark at index and the
-// same mark later in line. An opening mark is followed by a character that is
-// not a space, and a closing mark follows one. A single mark is not a part of a
-// double one. A mark of underscores does not open or close inside a word, so
-// snake_case stays as text. A code span inside the content is skipped whole.
-func telegramDelimited(line string, index int, mark, kind string) (TelegramMarkdownSpan, bool) {
-       char := mark[0]
-       after := index + len(mark)
-       if !strings.HasPrefix(line[index:], mark) || after >= len(line) || telegramIsSpace(line[after]) {
-               return TelegramMarkdownSpan{}, false
-       }
-
-       if len(mark) == 1 && (line[after] == char || index > 0 && line[index-1] == char) {
-               return TelegramMarkdownSpan{}, false
-       }
-
-       if char == '_' && index > 0 && telegramWordCharBefore(line[:index]) {
-               return TelegramMarkdownSpan{}, false
-       }
-
-       for closing := after + 1; closing < len(line); closing++ {
-               switch {
-               case line[closing] == '\\':
-                       closing++
-                       continue
-               case line[closing] == '`':
-                       span, found := telegramCodeSpan(line, closing)
-                       if found {
-                               closing = span.end - 1
-                       }
-
-                       continue
-               case !strings.HasPrefix(line[closing:], mark) || telegramIsSpace(line[closing-1]):
-                       continue
-               }
-
-               end := closing + len(mark)
-               if len(mark) == 1 && (line[closing-1] == char || end < len(line) && line[end] == char) {
-                       continue
-               }
-
-               if char == '_' && telegramWordChar(line[end:]) {
-                       continue
-               }
-
-               return TelegramMarkdownSpan{
-                       kind:    kind,
-                       content: line[after:closing],
-                       end:     end,
-               }, true
-       }
-
-       return TelegramMarkdownSpan{}, false
-}
-
-// telegramBackticks returns the length of the run of backticks at the start of
-// text.
-func telegramBackticks(text string) int {
-       return len(text) - len(strings.TrimLeft(text, "`"))
-}
-
-// telegramIsSpace tells if char is a space, a tab, or a line break.
-func telegramIsSpace(char byte) bool {
-       return char == ' ' || char == '\t' || char == '\n' || char == '\r'
-}
-
-// telegramWordCharBefore tells if the character that ends text is a letter or a
-// digit.
-func telegramWordCharBefore(text string) bool {
-       char, _ := utf8.DecodeLastRuneInString(text)
-       return text != "" && (unicode.IsLetter(char) || unicode.IsDigit(char))
-}

 
 

-// telegramWordChar tells if the character that starts text is a letter or a
-// digit. An underscore next to one is a part of a word, such as in snake_case.
-func telegramWordChar(text string) bool {
-       char, _ := utf8.DecodeRuneInString(text)
-       return text != "" && (unicode.IsLetter(char) || unicode.IsDigit(char))
+       out.WriteString(rewrite(line[prose:]))

 }
 }