Carry new images through model request chains

[onechat] / tools.go

diff --git a/tools.go b/tools.go
index 737618e91bd77851ebda0d658ca9dec0cda4d4a7..96cbb97d38010e9d6e53871b7bef0c471dba5dcf 100644 (file)
--- a/tools.go
+++ b/tools.go
@@ -49,30 +49,42 @@ type ToolChat interface {
 // the log as it happens: the call as a tool message, the result as an echo
 // message. The call is durable before its tool runs, so the log never misses an
 // action outside the agent. Commands and files run as the user in the working
 // the log as it happens: the call as a tool message, the result as an echo
 // message. The call is durable before its tool runs, so the log never misses an
 // action outside the agent. Commands and files run as the user in the working

-// directory, with no sandbox. Zoom with a name reads the log of a subagent.
-// The toolbox of the agent has the runs of its subagents for spawn. The
-// toolbox of a subagent has none, so spawn is an unknown tool there.
+// directory, with no sandbox. Zoom with a name reads a child's view or a
+// subagent's log. The toolbox of the agent has the runs of its subagents for
+// spawn and tell, the WhatsApp link for the WhatsApp tools, and the schedules
+// for the schedule tools. Each permanent agent has the parent-child queue. Only
+// the root has child creation and source routes. A child has neither of them
+// nor a WhatsApp link. The toolbox of a subagent has none of them, so these are
+// unknown tools there.

 type Toolbox struct {
        log       ToolLog
        chat      ToolChat
        subagents *Subagents
        runs      *SubagentRuns
 type Toolbox struct {
        log       ToolLog
        chat      ToolChat
        subagents *Subagents
        runs      *SubagentRuns

+       whatsapp  *WhatsApp
+       schedules *Schedules

        workdir   string
        workdir   string

+       agentName string
+       messages  *DaemonMessages
+       children  *DaemonChildren
+       routes    *Routes
+       images    *ImageStore

 }
 
 // Call logs the call of the tool name with its JSON arguments, makes the log
 }
 
 // Call logs the call of the tool name with its JSON arguments, makes the log

-// durable, runs the tool, and logs its clipped output, which it returns. A
-// failed tool gives its error as the output. An error of Call itself is an
-// error of the log. When ctx ends before the flush, the tool does not run.
-func (b *Toolbox) Call(ctx context.Context, name, arguments string) (string, error) {
+// durable, runs the tool, and logs its clipped text. It returns that text or
+// the text and image parts of a single-message zoom. Pixels never enter the
+// log. A failed tool gives its error as the output. An error of Call itself is
+// an error of the log. When ctx ends before the flush, the tool does not run.
+func (b *Toolbox) Call(ctx context.Context, name, arguments string) (any, error) {

        _, err := b.log.Append("tool", toolCallText(name, arguments))
        if err != nil {
        _, err := b.log.Append("tool", toolCallText(name, arguments))
        if err != nil {

-               return "", err
+               return "", fmt.Errorf("append tool call: %w", err)

        }
 
        err = b.log.Flush(ctx)
        if err != nil && ctx.Err() == nil {
        }
 
        err = b.log.Flush(ctx)
        if err != nil && ctx.Err() == nil {

-               return "", err
+               return "", fmt.Errorf("flush tool call: %w", err)

        }
 
        output := "not run: the turn was interrupted"
        }
 
        output := "not run: the turn was interrupted"


@@ -80,21 +92,39 @@ func (b *Toolbox) Call(ctx context.Context, name, arguments string) (string, err
                output = toolClip(b.run(ctx, name, arguments))
        }
 
                output = toolClip(b.run(ctx, name, arguments))
        }
 

+       var (
+               result    any = output
+               resultErr error
+       )
+
+       if err == nil {
+               result, resultErr = imageToolOutput(b, name, arguments, output)
+       }
+
+       if resultErr != nil {
+               output = "error: " + resultErr.Error()
+               result = output
+       }
+

        _, err = b.log.Append("echo", output)
        if err != nil {
        _, err = b.log.Append("echo", output)
        if err != nil {

-               return "", err
+               return "", fmt.Errorf("append tool result: %w", err)

        }
 
        }
 

-       return output, nil
+       return result, nil

 }
 
 // run runs one tool and returns its output, or its error as text.
 func (b *Toolbox) run(ctx context.Context, name, arguments string) string {
 }
 
 // run runs one tool and returns its output, or its error as text.
 func (b *Toolbox) run(ctx context.Context, name, arguments string) string {

-       var output string
-       var err error
+       var (
+               output string
+               err    error
+       )
+

        switch name {
        case "zoom":
                var args ToolZoomArguments
        switch name {
        case "zoom":
                var args ToolZoomArguments

+

                err = toolDecodeArguments(arguments, &args)
                switch {
                case err != nil:
                err = toolDecodeArguments(arguments, &args)
                switch {
                case err != nil:


@@ -105,36 +135,42 @@ func (b *Toolbox) run(ctx context.Context, name, arguments string) string {
                }
        case "date":
                var args ToolDateArguments
                }
        case "date":
                var args ToolDateArguments

+

                err = toolDecodeArguments(arguments, &args)
                if err == nil {
                        output, err = b.date(args.ID)
                }
        case "shell":
                var args ToolShellArguments
                err = toolDecodeArguments(arguments, &args)
                if err == nil {
                        output, err = b.date(args.ID)
                }
        case "shell":
                var args ToolShellArguments

+

                err = toolDecodeArguments(arguments, &args)
                if err == nil {
                        output = b.shell(ctx, args.Command)
                }
        case "read":
                var args ToolPathArguments
                err = toolDecodeArguments(arguments, &args)
                if err == nil {
                        output = b.shell(ctx, args.Command)
                }
        case "read":
                var args ToolPathArguments

+

                err = toolDecodeArguments(arguments, &args)
                if err == nil {
                        output, err = b.read(args.Path)
                }
        case "write":
                var args ToolWriteArguments
                err = toolDecodeArguments(arguments, &args)
                if err == nil {
                        output, err = b.read(args.Path)
                }
        case "write":
                var args ToolWriteArguments

+

                err = toolDecodeArguments(arguments, &args)
                if err == nil {
                        output, err = b.write(args.Path, args.Text)
                }
        case "edit":
                var args ToolEditArguments
                err = toolDecodeArguments(arguments, &args)
                if err == nil {
                        output, err = b.write(args.Path, args.Text)
                }
        case "edit":
                var args ToolEditArguments

+

                err = toolDecodeArguments(arguments, &args)
                if err == nil {
                        output, err = b.edit(args.Path, args.Old, args.New)
                }
        case "spawn":
                var args ToolSpawnArguments
                err = toolDecodeArguments(arguments, &args)
                if err == nil {
                        output, err = b.edit(args.Path, args.Old, args.New)
                }
        case "spawn":
                var args ToolSpawnArguments

+

                err = toolDecodeArguments(arguments, &args)
                switch {
                case b.runs == nil:
                err = toolDecodeArguments(arguments, &args)
                switch {
                case b.runs == nil:


@@ -142,6 +178,107 @@ func (b *Toolbox) run(ctx context.Context, name, arguments string) string {
                case err == nil:
                        output, err = b.runs.Spawn(ctx, args.Tasks, args.Effort)
                }
                case err == nil:
                        output, err = b.runs.Spawn(ctx, args.Tasks, args.Effort)
                }

+       case "agent_create":
+               var args ToolAgentCreateArguments
+
+               err = toolDecodeArguments(arguments, &args)
+               switch {
+               case b.children == nil:
+                       err = fmt.Errorf("unknown tool %q", name)
+               case err == nil:
+                       output, err = b.children.Create(args.Name, args.Responsibility)
+               }
+       case "route", "unroute", "routes":
+               var args ToolRouteArguments
+
+               switch name {
+               case "routes":
+                       err = toolDecodeArguments(arguments, &struct{}{})
+               case "unroute":
+                       var source ToolRouteSourceArguments
+
+                       err = toolDecodeArguments(arguments, &source)
+                       args.Source = source.Source
+               default:
+                       err = toolDecodeArguments(arguments, &args)
+               }
+
+               switch {
+               case b.routes == nil:
+                       err = fmt.Errorf("unknown tool %q", name)
+               case err != nil:
+               case name == "routes":
+                       output = b.routes.List()
+               case name == "route" && (b.messages == nil || b.messages.Child(b.agentName, args.Agent) == nil):
+                       err = errors.New("the agent must be a child of the root")
+               default:
+                       err = b.routes.Set(args.Source, args.Agent)
+                       output = "Route removed."
+                       if name == "route" {
+                               output = "Route set."
+                       }
+               }
+       case "tell":
+               var args ToolTellArguments
+
+               err = toolDecodeArguments(arguments, &args)
+               switch {
+               case b.runs == nil:
+                       err = fmt.Errorf("unknown tool %q", name)
+               case err == nil && b.messages != nil && b.messages.Child(b.agentName, args.Name) != nil:
+                       err = b.messages.Tell(b.agentName, args.Name, args.Message)
+                       output = "Sent. " + args.Name + "'s reply will reach you as a message."
+               case err == nil:
+                       output, err = b.runs.Tell(ctx, args.Name, args.Message)
+               }
+       case "whatsapp_find":
+               var args ToolWhatsAppFindArguments
+
+               err = toolDecodeArguments(arguments, &args)
+               switch {
+               case b.whatsapp == nil:
+                       err = fmt.Errorf("unknown tool %q", name)
+               case err == nil:
+                       output, err = b.whatsapp.Find(ctx, args.Name)
+               }
+       case "whatsapp_send":
+               var args ToolWhatsAppSendArguments
+
+               err = toolDecodeArguments(arguments, &args)
+               switch {
+               case b.whatsapp == nil:
+                       err = fmt.Errorf("unknown tool %q", name)
+               case err == nil:
+                       output, err = b.whatsapp.Send(ctx, args.ID, args.Chat, args.Text)
+               }
+       case "schedule_set":
+               var args ToolScheduleSetArguments
+
+               err = toolDecodeArguments(arguments, &args)
+               switch {
+               case b.schedules == nil:
+                       err = fmt.Errorf("unknown tool %q", name)
+               case err == nil:
+                       output, err = b.schedules.Set(args.Text, args.At, args.Cron)
+               }
+       case "schedule_list":
+               err = toolDecodeArguments(arguments, &struct{}{})
+               switch {
+               case b.schedules == nil:
+                       err = fmt.Errorf("unknown tool %q", name)
+               case err == nil:
+                       output = b.schedules.List()
+               }
+       case "schedule_remove":
+               var args ToolScheduleRemoveArguments
+
+               err = toolDecodeArguments(arguments, &args)
+               switch {
+               case b.schedules == nil:
+                       err = fmt.Errorf("unknown tool %q", name)
+               case err == nil:
+                       output, err = b.schedules.Remove(args.ID)
+               }

        default:
                err = fmt.Errorf("unknown tool %q", name)
        }
        default:
                err = fmt.Errorf("unknown tool %q", name)
        }


@@ -153,8 +290,15 @@ func (b *Toolbox) run(ctx context.Context, name, arguments string) string {
        return output
 }
 
        return output
 }
 

-// zoomAgent returns the whole log of the subagent name.
+// zoomAgent returns a child's view or the whole log of a subagent.

 func (b *Toolbox) zoomAgent(ctx context.Context, name string) (string, error) {
 func (b *Toolbox) zoomAgent(ctx context.Context, name string) (string, error) {

+       if b.messages != nil {
+               child := b.messages.Child(b.agentName, name)
+               if child != nil {
+                       return strings.Join(child.memory.ViewLines(nil), ""), nil
+               }
+       }
+

        subagent, err := b.subagents.Get(ctx, name)
        if err != nil {
                return "", err
        subagent, err := b.subagents.Get(ctx, name)
        if err != nil {
                return "", err


@@ -177,18 +321,20 @@ func (b *Toolbox) zoom(id, n int) (string, error) {
        if n == 1 {
                message, err := b.chat.Message(id)
                if err != nil {
        if n == 1 {
                message, err := b.chat.Message(id)
                if err != nil {

-                       return "", err
+                       return "", fmt.Errorf("read zoom message: %w", err)

                }
 
                return leafSource(message), nil
        }
 
        level := bits.TrailingZeros(uint(n)) - 1
                }
 
                return leafSource(message), nil
        }
 
        level := bits.TrailingZeros(uint(n)) - 1

+

        var lines []string
        var lines []string

+

        for _, index := range []int{2 * id / n, 2*id/n + 1} {
                node, built, err := b.chat.Node(level, index)
                if err != nil {
        for _, index := range []int{2 * id / n, 2*id/n + 1} {
                node, built, err := b.chat.Node(level, index)
                if err != nil {

-                       return "", err
+                       return "", fmt.Errorf("read zoom node: %w", err)

                }
 
                text := viewPending
                }
 
                text := viewPending


@@ -210,12 +356,12 @@ func (b *Toolbox) date(id int) (string, error) {
 
        message, err := b.chat.Message(id)
        if err != nil {
 
        message, err := b.chat.Message(id)
        if err != nil {

-               return "", err
+               return "", fmt.Errorf("read dated message: %w", err)

        }
 
        date, err := time.Parse(time.RFC3339Nano, message.Date)
        if err != nil {
        }
 
        date, err := time.Parse(time.RFC3339Nano, message.Date)
        if err != nil {

-               return "", err
+               return "", fmt.Errorf("parse message date: %w", err)

        }
 
        return date.Local().Format("Monday 2006-01-02 15:04:05 -07:00"), nil
        }
 
        return date.Local().Format("Monday 2006-01-02 15:04:05 -07:00"), nil


@@ -241,7 +387,9 @@ func (b *Toolbox) shell(ctx context.Context, command string) string {
        err := cmd.Run()
 
        text := output.String()
        err := cmd.Run()
 
        text := output.String()

+

        var exit *exec.ExitError
        var exit *exec.ExitError

+

        switch {
        case ctx.Err() != nil:
                text += "\n[interrupted]"
        switch {
        case ctx.Err() != nil:
                text += "\n[interrupted]"


@@ -260,7 +408,7 @@ func (b *Toolbox) shell(ctx context.Context, command string) string {
 func (b *Toolbox) read(path string) (text string, err error) {
        file, err := os.Open(b.path(path))
        if err != nil {
 func (b *Toolbox) read(path string) (text string, err error) {
        file, err := os.Open(b.path(path))
        if err != nil {

-               return "", err
+               return "", fmt.Errorf("open tool file: %w", err)

        }
 
        defer func() {
        }
 
        defer func() {


@@ -270,7 +418,7 @@ func (b *Toolbox) read(path string) (text string, err error) {
        output := &ToolHeadTail{}
        _, err = io.Copy(output, file)
        if err != nil {
        output := &ToolHeadTail{}
        _, err = io.Copy(output, file)
        if err != nil {

-               return "", err
+               return "", fmt.Errorf("read tool file: %w", err)

        }
 
        return output.String(), nil
        }
 
        return output.String(), nil


@@ -282,12 +430,12 @@ func (b *Toolbox) write(path, text string) (string, error) {
        path = b.path(path)
        err := os.MkdirAll(filepath.Dir(path), 0o755)
        if err != nil {
        path = b.path(path)
        err := os.MkdirAll(filepath.Dir(path), 0o755)
        if err != nil {

-               return "", err
+               return "", fmt.Errorf("create tool file directory: %w", err)

        }
 
        err = os.WriteFile(path, []byte(text), 0o644)
        if err != nil {
        }
 
        err = os.WriteFile(path, []byte(text), 0o644)
        if err != nil {

-               return "", err
+               return "", fmt.Errorf("write tool file: %w", err)

        }
 
        return fmt.Sprintf("wrote %d bytes to %s", len(text), path), nil
        }
 
        return fmt.Sprintf("wrote %d bytes to %s", len(text), path), nil


@@ -299,7 +447,7 @@ func (b *Toolbox) edit(path, old, replacement string) (string, error) {
        path = b.path(path)
        data, err := os.ReadFile(path)
        if err != nil {
        path = b.path(path)
        data, err := os.ReadFile(path)
        if err != nil {

-               return "", err
+               return "", fmt.Errorf("read file to edit: %w", err)

        }
 
        if old == "" {
        }
 
        if old == "" {


@@ -313,12 +461,12 @@ func (b *Toolbox) edit(path, old, replacement string) (string, error) {
 
        info, err := os.Stat(path)
        if err != nil {
 
        info, err := os.Stat(path)
        if err != nil {

-               return "", err
+               return "", fmt.Errorf("read file mode: %w", err)

        }
 
        err = os.WriteFile(path, []byte(strings.Replace(string(data), old, replacement, 1)), info.Mode().Perm())
        if err != nil {
        }
 
        err = os.WriteFile(path, []byte(strings.Replace(string(data), old, replacement, 1)), info.Mode().Perm())
        if err != nil {

-               return "", err
+               return "", fmt.Errorf("write edited file: %w", err)

        }
 
        return "edited " + path, nil
        }
 
        return "edited " + path, nil


@@ -334,8 +482,9 @@ func (b *Toolbox) path(path string) string {
        return filepath.Join(b.workdir, path)
 }
 
        return filepath.Join(b.workdir, path)
 }
 

-// ToolZoomArguments are the arguments of zoom. A Name that is not nil names
-// the subagent whose whole log zoom gives, and then ID and N are not used.
+// ToolZoomArguments are the arguments of zoom. A Name that is not nil names the
+// child whose view or the subagent whose whole log zoom gives. ID and N are
+// then not used.

 type ToolZoomArguments struct {
        ID   int     `json:"id"`
        N    int     `json:"n"`
 type ToolZoomArguments struct {
        ID   int     `json:"id"`
        N    int     `json:"n"`


@@ -377,6 +526,54 @@ type ToolSpawnArguments struct {
        Effort *string  `json:"effort"`
 }
 
        Effort *string  `json:"effort"`
 }
 

+// ToolAgentCreateArguments are the arguments of agent_create.
+type ToolAgentCreateArguments struct {
+       Name           string `json:"name"`
+       Responsibility string `json:"responsibility"`
+}
+
+// ToolRouteArguments are the arguments of route.
+type ToolRouteArguments struct {
+       Source string `json:"source"`
+       Agent  string `json:"agent"`
+}
+
+// ToolRouteSourceArguments are the arguments of unroute.
+type ToolRouteSourceArguments struct {
+       Source string `json:"source"`
+}
+
+// ToolTellArguments are the arguments of tell.
+type ToolTellArguments struct {
+       Name    string `json:"name"`
+       Message string `json:"message"`
+}
+
+// ToolWhatsAppFindArguments are the arguments of whatsapp_find.
+type ToolWhatsAppFindArguments struct {
+       Name string `json:"name"`
+}
+
+// ToolWhatsAppSendArguments are the arguments of whatsapp_send.
+type ToolWhatsAppSendArguments struct {
+       ID   string `json:"id"`
+       Chat string `json:"chat"`
+       Text string `json:"text"`
+}
+
+// ToolScheduleSetArguments are the arguments of schedule_set. Exactly one of At
+// and Cron is not nil.
+type ToolScheduleSetArguments struct {
+       Text string  `json:"text"`
+       At   *string `json:"at"`
+       Cron *string `json:"cron"`
+}
+
+// ToolScheduleRemoveArguments are the arguments of schedule_remove.
+type ToolScheduleRemoveArguments struct {
+       ID int `json:"id"`
+}
+

 // ToolHeadTail keeps the first and the last toolClipBytes bytes written to it,
 // and counts the bytes between them that it drops.
 type ToolHeadTail struct {
 // ToolHeadTail keeps the first and the last toolClipBytes bytes written to it,
 // and counts the bytes between them that it drops.
 type ToolHeadTail struct {


@@ -442,6 +639,7 @@ func toolClip(text string) string {
 func toolDecodeArguments(arguments string, args any) error {
        decoder := json.NewDecoder(strings.NewReader(arguments))
        decoder.DisallowUnknownFields()
 func toolDecodeArguments(arguments string, args any) error {
        decoder := json.NewDecoder(strings.NewReader(arguments))
        decoder.DisallowUnknownFields()

+

        err := decoder.Decode(args)
        if err != nil {
                return fmt.Errorf("arguments: %w", err)
        err := decoder.Decode(args)
        if err != nil {
                return fmt.Errorf("arguments: %w", err)