git.giacomolli.uk - onechat/blob - image_model.go

   1 package main
   2
   3 import (
   4         "encoding/base64"
   5         "errors"
   6         "fmt"
   7         "strings"
   8 )
   9
  10 // imageUserMessage attaches pixels only to messages selected as new by a turn.
  11 func imageUserMessage(store *ImageStore, text string) (any, error) {
  12         names := imageFiles(text)
  13         if len(names) == 0 {
  14                 return modelUserMessage(text), nil
  15         }
  16
  17         parts, err := imageParts(store, text, names)
  18         if err != nil {
  19                 return nil, fmt.Errorf("make image message: %w", err)
  20         }
  21
  22         return modelUserParts(parts), nil
  23 }
  24
  25 // imageParts is the shared wire format for image messages and zoom results.
  26 // Tool results use this list directly as function_call_output.output.
  27 func imageParts(store *ImageStore, text string, names []string) ([]ModelInputPart, error) {
  28         parts := []ModelInputPart{{
  29                 Type: "input_text",
  30                 Text: text,
  31         }}
  32         for _, name := range names {
  33                 if store == nil {
  34                         return nil, errors.New("image store is not available")
  35                 }
  36
  37                 data, err := store.Read(name)
  38                 if err != nil {
  39                         return nil, fmt.Errorf("read image part: %w", err)
  40                 }
  41
  42                 mime := "image/" + strings.TrimPrefix(name[64:], ".")
  43                 if strings.HasSuffix(name, ".jpg") {
  44                         mime = "image/jpeg"
  45                 }
  46
  47                 parts = append(parts, ModelInputPart{
  48                         Type:     "input_image",
  49                         ImageURL: "data:" + mime + ";base64," + base64.StdEncoding.EncodeToString(data),
  50                 })
  51         }
  52
  53         return parts, nil
  54 }
  55
  56 // imageToolOutput leaves all results as text except zoom of one image message.
  57 // Only the text goes to the log, never the data URLs.
  58 func imageToolOutput(box *Toolbox, name, arguments, text string) (any, error) {
  59         if name != "zoom" || strings.HasPrefix(text, "error:") {
  60                 return text, nil
  61         }
  62
  63         var args ToolZoomArguments
  64
  65         err := toolDecodeArguments(arguments, &args)
  66         if err != nil {
  67                 return nil, fmt.Errorf("decode image zoom: %w", err)
  68         }
  69
  70         if args.Name != nil || args.N != 1 {
  71                 return text, nil
  72         }
  73
  74         message, err := box.chat.Message(args.ID)
  75         if err != nil {
  76                 return nil, fmt.Errorf("read image zoom: %w", err)
  77         }
  78
  79         names := imageFiles(message.Text)
  80         if len(names) == 0 {
  81                 return text, nil
  82         }
  83
  84         parts, err := imageParts(box.images, text, names)
  85         if err != nil {
  86                 return nil, fmt.Errorf("make image zoom: %w", err)
  87         }
  88
  89         return parts, nil
  90 }