git.giacomolli.uk - onechat/blob - compactor.go

   1 package main
   2
   3 import (
   4         "context"
   5         "errors"
   6         "fmt"
   7         "strings"
   8         "unicode/utf8"
   9 )
  10
  11 // compactorTries is the most calls for one node.
  12 const compactorTries = 5
  13
  14 // compactorRuler shows the model the size limit of a line. Models cannot count
  15 // bytes, and a sample line as the ruler gets its content copied.
  16 var compactorRuler = strings.Repeat("-", nodeBytes)
  17
  18 // Compactor builds the nodes of one agent with a model. A compaction is a call
  19 // like a turn, with the same system prompt and tools, so it reads them from the
  20 // prompt cache of the turns. The tools are never called. After them comes one
  21 // user message with the compaction view up to the node, its closing tag, a
  22 // newline, and the task.
  23 //
  24 // When the line is over nodeBytes, the next try sends the same message, plus
  25 // one more user message with a note with the size of the newest line and its
  26 // cut at the limit. Only the first call of an image leaf sends its pixels. A
  27 // try never sends the model's earlier reply or reasoning back, so every try
  28 // begins with the same view and task and has about the same size. After at most
  29 // compactorTries tries, the shortest line is the node. A few bytes over the
  30 // limit is fine, because the view measures the real sizes. A call that fails
  31 // before any line fails the build, and the core tries it again at the next
  32 // message.
  33 type Compactor struct {
  34         client       *ModelClient
  35         model        ModelConfig
  36         instructions func() (string, error)
  37         tools        []ModelTool
  38         images       *ImageStore
  39 }
  40
  41 // newCompactor returns the compactor of an agent, which sends its calls through
  42 // client with model. instructions gives the system prompt of the agent at each
  43 // build. tools are the tools of that agent, so a build sends the same prompt
  44 // and tools as its turns.
  45 func newCompactor(client *ModelClient, instructions func() (string, error), model ModelConfig, tools []ModelTool, images *ImageStore) *Compactor {
  46         return &Compactor{
  47                 client:       client,
  48                 model:        model,
  49                 instructions: instructions,
  50                 tools:        tools,
  51                 images:       images,
  52         }
  53 }
  54
  55 // Summarize returns the line of the node of task.
  56 func (c *Compactor) Summarize(ctx context.Context, task Task) (string, error) {
  57         text, err := compactorTask(task)
  58         if err != nil {
  59                 return "", err
  60         }
  61
  62         instructions, err := c.instructions()
  63         if err != nil {
  64                 return "", err
  65         }
  66
  67         source := task.Context + "\n" + text
  68         parts, err := imageParts(c.images, source, task.Images)
  69         if err != nil {
  70                 return "", fmt.Errorf("read compaction images: %w", err)
  71         }
  72
  73         first := []any{modelUserParts(parts)}
  74
  75         var lines []string
  76
  77         note := ""
  78         for len(lines) < compactorTries {
  79                 input := first
  80                 if note != "" {
  81                         input = []any{modelUserMessage(source), modelUserMessage(note)}
  82                 }
  83
  84                 var response ModelResponse
  85
  86                 response, err = c.client.Create(ctx, ModelRequest{
  87                         Model:        c.model.Model,
  88                         Effort:       c.model.Effort,
  89                         Instructions: instructions,
  90                         Tools:        c.tools,
  91                         Input:        input,
  92                         Node:         &NodeKey{task.Level, task.Index},
  93                 })
  94                 if err != nil {
  95                         break
  96                 }
  97
  98                 line := strings.TrimSpace(response.Text)
  99                 if line == "" {
 100                         err = errors.New("compaction gave no line")
 101                         break
 102                 }
 103
 104                 lines = append(lines, line)
 105                 if len(line) <= nodeBytes {
 106                         break
 107                 }
 108
 109                 note = compactorTooLong(line)
 110         }
 111
 112         if len(lines) == 0 {
 113                 return "", err
 114         }
 115
 116         return compactorShortest(lines), nil
 117 }
 118
 119 // compactorTask returns the task of a build, with its input.
 120 func compactorTask(task Task) (string, error) {
 121         if task.Level == 0 && len(task.Input) == 1 {
 122                 text := fmt.Sprintf(`Compaction: compress message %d into one line of at most %d bytes
 123 (about 70 words), the length of this ruler:
 124 %s
 125 <input>
 126 %s
 127 </input>`, task.Index, nodeBytes, compactorRuler, task.Input[0])
 128
 129                 if len(task.Images) > 0 {
 130                         text += "\nKeep each [image FILE] name and describe its kind, visible text, amounts, names, and context."
 131                 }
 132
 133                 return text, nil
 134         }
 135
 136         if task.Level > 0 && len(task.Input) == 2 {
 137                 first := task.Index << task.Level
 138                 half := 1 << (task.Level - 1)
 139
 140                 return fmt.Sprintf(`Compaction: merge lines %d+%d and %d+%d, adjacent, into one line of at most
 141 %d bytes (about 70 words), the length of this ruler:
 142 %s
 143 <chat> may hold their messages, %d to %d, in more detail: take details
 144 of them from there too.
 145 <input>
 146 %s
 147 %s
 148 </input>`, first, half, first+half, half, nodeBytes, compactorRuler, first, nodeEnd(task.Level, task.Index)-1, task.Input[0], task.Input[1]), nil
 149         }
 150
 151         return "", fmt.Errorf("node (%d, %d) has %d inputs", task.Level, task.Index, len(task.Input))
 152 }
 153
 154 // compactorTooLong returns the note for a line over nodeBytes: its size and its
 155 // first nodeBytes bytes, which show the model where the limit cuts it.
 156 func compactorTooLong(line string) string {
 157         return fmt.Sprintf(`Too long: your last line for this <input> was %d bytes,
 158 over the %d-byte limit. Write the whole line again, cutting just
 159 enough of the least valuable items to fit before this cut:
 160 %s| <- LIMIT`, len(line), nodeBytes, compactorCutLine(line))
 161 }
 162
 163 // compactorShortest returns the first of the compactorShortest lines.
 164 func compactorShortest(lines []string) string {
 165         best := lines[0]
 166         for _, line := range lines[1:] {
 167                 if len(line) < len(best) {
 168                         best = line
 169                 }
 170         }
 171
 172         return best
 173 }
 174
 175 // compactorCutLine returns the first nodeBytes bytes of line, without a part of
 176 // a character at the end.
 177 func compactorCutLine(line string) string {
 178         if len(line) <= nodeBytes {
 179                 return line
 180         }
 181
 182         cut := line[:nodeBytes]
 183         for !utf8.ValidString(cut) {
 184                 cut = cut[:len(cut)-1]
 185         }
 186
 187         return cut
 188 }