git.giacomolli.uk - onechat/blob - compactor.go

   1 package main
   2
   3 import (
   4         "context"
   5         "errors"
   6         "fmt"
   7         "slices"
   8         "strings"
   9         "unicode/utf8"
  10 )
  11
  12 // compactorTries is the most calls for one node.
  13 const compactorTries = 5
  14
  15 // compactorBlock is the number of view lines in one block message.
  16 const compactorBlock = 4
  17
  18 // compactorRuler shows the model the size limit of a line. Models cannot count
  19 // bytes, and a sample line as the ruler gets its content copied.
  20 var compactorRuler = strings.Repeat("-", nodeBytes)
  21
  22 // newCompactor returns the compactor of an agent, which sends its calls through
  23 // client with model. instructions gives the system prompt of the agent at each
  24 // build, so a build sends the same prompt as the turns of that time.
  25 func newCompactor(client *ModelClient, instructions func() (string, error), model ModelConfig) *Compactor {
  26         return &Compactor{
  27                 client:       client,
  28                 model:        model,
  29                 instructions: instructions,
  30                 tools:        promptTools(),
  31         }
  32 }
  33
  34 // Compactor builds the nodes of one agent with a model. A compaction is a call
  35 // like a turn, with the same system prompt and tools, so it reads them from the
  36 // prompt cache of the turns. The tools are never called. After them come the
  37 // compaction view up to the node and the task, as the user messages that
  38 // compactorTexts gives. Each whole block of compactorBlock view lines is its
  39 // own message, and the last message holds the rest of the lines and the task.
  40 // The backend reads a cache entry only at a message end of a request, so a
  41 // compaction whose view begins with the same lines as an earlier one begins
  42 // with the same block messages.
  43 //
  44 // When the line is over nodeBytes, the next try sends the same messages, plus
  45 // one more user message with a note with the size of the newest line and its
  46 // cut at the limit. A try never sends the model's earlier reply or reasoning
  47 // back, so every try begins with the whole earlier try and has about the same
  48 // size. After at most compactorTries tries, the shortest line is the node. A
  49 // few bytes over the limit is fine, because the view measures the real sizes. A
  50 // call that fails before any line fails the build, and the core tries it again
  51 // at the next message.
  52 type Compactor struct {
  53         client       *ModelClient
  54         model        ModelConfig
  55         instructions func() (string, error)
  56         tools        []ModelTool
  57 }
  58
  59 // Summarize returns the line of the node of task.
  60 func (c *Compactor) Summarize(ctx context.Context, task Task) (string, error) {
  61         text, err := compactorTask(task)
  62         if err != nil {
  63                 return "", err
  64         }
  65
  66         instructions, err := c.instructions()
  67         if err != nil {
  68                 return "", err
  69         }
  70
  71         var first []any
  72
  73         for _, message := range compactorTexts(task.Context, text) {
  74                 first = append(first, modelUserMessage(message))
  75         }
  76
  77         var lines []string
  78
  79         note := ""
  80         for len(lines) < compactorTries {
  81                 input := first
  82                 if note != "" {
  83                         input = append(slices.Clip(first), modelUserMessage(note))
  84                 }
  85
  86                 var response ModelResponse
  87
  88                 response, err = c.client.Create(ctx, ModelRequest{
  89                         Model:        c.model.Model,
  90                         Effort:       c.model.Effort,
  91                         Instructions: instructions,
  92                         Tools:        c.tools,
  93                         Input:        input,
  94                         Node:         &NodeKey{task.Level, task.Index},
  95                 })
  96                 if err != nil {
  97                         break
  98                 }
  99
 100                 line := strings.TrimSpace(response.Text)
 101                 if line == "" {
 102                         err = errors.New("compaction gave no line")
 103                         break
 104                 }
 105
 106                 lines = append(lines, line)
 107                 if len(line) <= nodeBytes {
 108                         break
 109                 }
 110
 111                 note = compactorTooLong(line)
 112         }
 113
 114         if len(lines) == 0 {
 115                 return "", err
 116         }
 117
 118         return compactorShortest(lines), nil
 119 }
 120
 121 // compactorTask returns the task of a build, with its input.
 122 func compactorTask(task Task) (string, error) {
 123         if task.Level == 0 && len(task.Input) == 1 {
 124                 return fmt.Sprintf(`Compaction: compress message %d into one line of at most %d bytes
 125 (about 70 words), the length of this ruler:
 126 %s
 127 <input>
 128 %s
 129 </input>`, task.Index, nodeBytes, compactorRuler, task.Input[0]), nil
 130         }
 131
 132         if task.Level > 0 && len(task.Input) == 2 {
 133                 first := task.Index << task.Level
 134                 half := 1 << (task.Level - 1)
 135
 136                 return fmt.Sprintf(`Compaction: merge lines %d+%d and %d+%d, adjacent, into one line of at most
 137 %d bytes (about 70 words), the length of this ruler:
 138 %s
 139 <chat> may hold their messages, %d to %d, in more detail: take details
 140 of them from there too.
 141 <input>
 142 %s
 143 %s
 144 </input>`, first, half, first+half, half, nodeBytes, compactorRuler, first, nodeEnd(task.Level, task.Index)-1, task.Input[0], task.Input[1]), nil
 145         }
 146
 147         return "", fmt.Errorf("node (%d, %d) has %d inputs", task.Level, task.Index, len(task.Input))
 148 }
 149
 150 // compactorTexts returns the texts of the user messages of a first try for the
 151 // compaction view and the task text. Each whole block of compactorBlock lines,
 152 // counted from the first line of the view, is one message. The last message
 153 // holds the rest of the lines, the closing tag, a newline, and the task. The
 154 // first message starts with the opening tag. Joined, the texts are the view, a
 155 // newline, and the task.
 156 func compactorTexts(view, task string) []string {
 157         body := strings.TrimSuffix(strings.TrimPrefix(view, "<chat>\n"), "</chat>")
 158         lines := strings.SplitAfter(body, "\n")
 159         lines = lines[:len(lines)-1]
 160         whole := len(lines) - len(lines)%compactorBlock
 161
 162         var texts []string
 163
 164         for n := 0; n < whole; n += compactorBlock {
 165                 texts = append(texts, viewMessage(n == 0, lines[n:n+compactorBlock]))
 166         }
 167
 168         return append(texts, viewMessage(whole == 0, lines[whole:])+"</chat>\n"+task)
 169 }
 170
 171 // compactorTooLong returns the note for a line over nodeBytes: its size and its
 172 // first nodeBytes bytes, which show the model where the limit cuts it.
 173 func compactorTooLong(line string) string {
 174         return fmt.Sprintf(`Too long: your last line for this <input> was %d bytes,
 175 over the %d-byte limit. Write the whole line again, cutting just
 176 enough of the least valuable items to fit before this cut:
 177 %s| <- LIMIT`, len(line), nodeBytes, compactorCutLine(line))
 178 }
 179
 180 // compactorShortest returns the first of the compactorShortest lines.
 181 func compactorShortest(lines []string) string {
 182         best := lines[0]
 183         for _, line := range lines[1:] {
 184                 if len(line) < len(best) {
 185                         best = line
 186                 }
 187         }
 188
 189         return best
 190 }
 191
 192 // compactorCutLine returns the first nodeBytes bytes of line, without a part of
 193 // a character at the end.
 194 func compactorCutLine(line string) string {
 195         if len(line) <= nodeBytes {
 196                 return line
 197         }
 198
 199         cut := line[:nodeBytes]
 200         for !utf8.ValidString(cut) {
 201                 cut = cut[:len(cut)-1]
 202         }
 203
 204         return cut
 205 }