git.giacomolli.uk - onechat/blob - image_model.go

   1 package main
   2
   3 import (
   4         "encoding/base64"
   5         "errors"
   6         "fmt"
   7         "strings"
   8 )
   9
  10 // imageUserMessage attaches pixels only to messages selected as new by a turn.
  11 func imageUserMessage(store *ImageStore, text string) (any, error) {
  12         names := imageFiles(text)
  13         if len(names) == 0 {
  14                 return modelUserMessage(text), nil
  15         }
  16
  17         parts, err := imageParts(store, text, names)
  18         if err != nil {
  19                 return nil, fmt.Errorf("make image message: %w", err)
  20         }
  21
  22         return modelUserParts(parts), nil
  23 }
  24
  25 // imageParts is the shared wire format for image messages and zoom results.
  26 // Tool results use this list directly as function_call_output.output.
  27 func imageParts(store *ImageStore, text string, names []string) ([]ModelInputPart, error) {
  28         parts := []ModelInputPart{{
  29                 Type: "input_text",
  30                 Text: text,
  31         }}
  32
  33         for _, name := range names {
  34                 if store == nil {
  35                         return nil, errors.New("image store is not available")
  36                 }
  37
  38                 data, err := store.Read(name)
  39                 if err != nil {
  40                         return nil, fmt.Errorf("read image part: %w", err)
  41                 }
  42
  43                 mime := "image/" + strings.TrimPrefix(name[64:], ".")
  44                 if strings.HasSuffix(name, ".jpg") {
  45                         mime = "image/jpeg"
  46                 }
  47
  48                 parts = append(parts, ModelInputPart{
  49                         Type:     "input_image",
  50                         ImageURL: "data:" + mime + ";base64," + base64.StdEncoding.EncodeToString(data),
  51                 })
  52         }
  53
  54         return parts, nil
  55 }
  56
  57 // imageToolOutput leaves all results as text except zoom of one image message.
  58 // Only the text goes to the log, never the data URLs.
  59 func imageToolOutput(box *Toolbox, name, arguments, text string) (any, error) {
  60         if name != "zoom" || strings.HasPrefix(text, "error:") {
  61                 return text, nil
  62         }
  63
  64         var args ToolZoomArguments
  65
  66         err := toolDecodeArguments(arguments, &args)
  67         if err != nil {
  68                 return nil, fmt.Errorf("decode image zoom: %w", err)
  69         }
  70
  71         if args.Name != nil || args.N != 1 {
  72                 return text, nil
  73         }
  74
  75         message, err := box.chat.Message(args.ID)
  76         if err != nil {
  77                 return nil, fmt.Errorf("read image zoom: %w", err)
  78         }
  79
  80         names := imageFiles(message.Text)
  81         if len(names) == 0 {
  82                 return text, nil
  83         }
  84
  85         parts, err := imageParts(box.images, text, names)
  86         if err != nil {
  87                 return nil, fmt.Errorf("make image zoom: %w", err)
  88         }
  89
  90         return parts, nil
  91 }