git.giacomolli.uk - onechat/blob - compactor.go

   1 package main
   2
   3 import (
   4         "context"
   5         "errors"
   6         "fmt"
   7         "strings"
   8         "unicode/utf8"
   9 )
  10
  11 // compactorTries is the most calls for one node.
  12 const compactorTries = 5
  13
  14 // compactorRuler shows the model the size limit of a line. Models cannot count
  15 // bytes, and a sample line as the ruler gets its content copied.
  16 var compactorRuler = strings.Repeat("-", nodeBytes)
  17
  18 // newCompactor returns the compactor of an agent, which sends its calls through
  19 // client with model. instructions gives the system prompt of the agent at each
  20 // build. tools are the tools of that agent, so a build sends the same prompt
  21 // and tools as its turns.
  22 func newCompactor(client *ModelClient, instructions func() (string, error), model ModelConfig, tools []ModelTool, images *ImageStore) *Compactor {
  23         return &Compactor{
  24                 client:       client,
  25                 model:        model,
  26                 instructions: instructions,
  27                 tools:        tools,
  28                 images:       images,
  29         }
  30 }
  31
  32 // Compactor builds the nodes of one agent with a model. A compaction is a call
  33 // like a turn, with the same system prompt and tools, so it reads them from the
  34 // prompt cache of the turns. The tools are never called. After them comes one
  35 // user message with the compaction view up to the node, its closing tag, a
  36 // newline, and the task.
  37 //
  38 // When the line is over nodeBytes, the next try sends the same message, plus
  39 // one more user message with a note with the size of the newest line and its
  40 // cut at the limit. Only the first call of an image leaf sends its pixels. A
  41 // try never sends the model's earlier reply or reasoning back, so every try
  42 // begins with the same view and task and has about the same size. After at most
  43 // compactorTries tries, the shortest line is the node. A few bytes over the
  44 // limit is fine, because the view measures the real sizes. A call that fails
  45 // before any line fails the build, and the core tries it again at the next
  46 // message.
  47 type Compactor struct {
  48         client       *ModelClient
  49         model        ModelConfig
  50         instructions func() (string, error)
  51         tools        []ModelTool
  52         images       *ImageStore
  53 }
  54
  55 // Summarize returns the line of the node of task.
  56 func (c *Compactor) Summarize(ctx context.Context, task Task) (string, error) {
  57         text, err := compactorTask(task)
  58         if err != nil {
  59                 return "", err
  60         }
  61
  62         instructions, err := c.instructions()
  63         if err != nil {
  64                 return "", err
  65         }
  66
  67         source := task.Context + "\n" + text
  68         parts, err := imageParts(c.images, source, task.Images)
  69         if err != nil {
  70                 return "", fmt.Errorf("read compaction images: %w", err)
  71         }
  72
  73         first := []any{modelUserParts(parts)}
  74
  75         var lines []string
  76
  77         note := ""
  78         for len(lines) < compactorTries {
  79                 input := first
  80                 if note != "" {
  81                         input = []any{modelUserMessage(source), modelUserMessage(note)}
  82                 }
  83
  84                 var response ModelResponse
  85
  86                 response, err = c.client.Create(ctx, ModelRequest{
  87                         Model:        c.model.Model,
  88                         Effort:       c.model.Effort,
  89                         Instructions: instructions,
  90                         Tools:        c.tools,
  91                         Input:        input,
  92                         Node:         &NodeKey{task.Level, task.Index},
  93                 })
  94                 if err != nil {
  95                         break
  96                 }
  97
  98                 line := strings.TrimSpace(response.Text)
  99                 if line == "" {
 100                         err = errors.New("compaction gave no line")
 101                         break
 102                 }
 103
 104                 lines = append(lines, line)
 105                 if len(line) <= nodeBytes {
 106                         break
 107                 }
 108
 109                 note = compactorTooLong(line)
 110         }
 111
 112         if len(lines) == 0 {
 113                 return "", err
 114         }
 115
 116         return compactorShortest(lines), nil
 117 }
 118
 119 // compactorTask returns the task of a build, with its input.
 120 func compactorTask(task Task) (string, error) {
 121         if task.Level == 0 && len(task.Input) == 1 {
 122                 text := fmt.Sprintf(`Compaction: compress message %d into one line of at most %d bytes
 123 (about 70 words), the length of this ruler:
 124 %s
 125 <input>
 126 %s
 127 </input>`, task.Index, nodeBytes, compactorRuler, task.Input[0])
 128                 if len(task.Images) > 0 {
 129                         text += "\nKeep each [image FILE] name and describe its kind, visible text, amounts, names, and context."
 130                 }
 131
 132                 return text, nil
 133         }
 134
 135         if task.Level > 0 && len(task.Input) == 2 {
 136                 first := task.Index << task.Level
 137                 half := 1 << (task.Level - 1)
 138
 139                 return fmt.Sprintf(`Compaction: merge lines %d+%d and %d+%d, adjacent, into one line of at most
 140 %d bytes (about 70 words), the length of this ruler:
 141 %s
 142 <chat> may hold their messages, %d to %d, in more detail: take details
 143 of them from there too.
 144 <input>
 145 %s
 146 %s
 147 </input>`, first, half, first+half, half, nodeBytes, compactorRuler, first, nodeEnd(task.Level, task.Index)-1, task.Input[0], task.Input[1]), nil
 148         }
 149
 150         return "", fmt.Errorf("node (%d, %d) has %d inputs", task.Level, task.Index, len(task.Input))
 151 }
 152
 153 // compactorTooLong returns the note for a line over nodeBytes: its size and its
 154 // first nodeBytes bytes, which show the model where the limit cuts it.
 155 func compactorTooLong(line string) string {
 156         return fmt.Sprintf(`Too long: your last line for this <input> was %d bytes,
 157 over the %d-byte limit. Write the whole line again, cutting just
 158 enough of the least valuable items to fit before this cut:
 159 %s| <- LIMIT`, len(line), nodeBytes, compactorCutLine(line))
 160 }
 161
 162 // compactorShortest returns the first of the compactorShortest lines.
 163 func compactorShortest(lines []string) string {
 164         best := lines[0]
 165         for _, line := range lines[1:] {
 166                 if len(line) < len(best) {
 167                         best = line
 168                 }
 169         }
 170
 171         return best
 172 }
 173
 174 // compactorCutLine returns the first nodeBytes bytes of line, without a part of
 175 // a character at the end.
 176 func compactorCutLine(line string) string {
 177         if len(line) <= nodeBytes {
 178                 return line
 179         }
 180
 181         cut := line[:nodeBytes]
 182         for !utf8.ValidString(cut) {
 183                 cut = cut[:len(cut)-1]
 184         }
 185
 186         return cut
 187 }