From: Anderson A. Giacomolli Date: Sat, 7 Nov 2026 16:52:39 +0000 (-0300) Subject: Name and order the probe code X-Git-Url: https://git.giacomolli.uk/onechat/commitdiff_plain/ec38dadaaf127d8bd2ffb72df450e8d3f5ac88c0?ds=sidebyside;hp=503486b8364f07ef44c06bcc1be566f62600b0eb Name and order the probe code Every package-level name of the probe starts with probe: its run ProbeRun, the summarizer ProbeSlotSummarizer that limits the builds at once, the reader of its log and of the usage file, and the parts of its report. Names such as ratio, timeText, and modelLines read as helpers of the whole program, but only the probe uses them. The declarations of the probe files follow the order of the go-code skill. The behavior does not change. Issue: 947bddb --- diff --git a/probe.go b/probe.go index e12516d..0b809e8 100644 --- a/probe.go +++ b/probe.go @@ -62,7 +62,7 @@ func probe(ctx context.Context, args []string, out io.Writer) (err error) { } } - entries, err := readProbeLog(args[0], count) + entries, err := probeReadLog(args[0], count) if err != nil { return err } @@ -100,7 +100,7 @@ func probe(ctx context.Context, args []string, out io.Writer) (err error) { usagePath := filepath.Join(root, "usage.jsonl") client := newModelClient(address, auth, "onechat-"+probeName, usagePath) client.LimitCalls(probeCallLimit) - summarizer := slotSummarizer{ + summarizer := ProbeSlotSummarizer{ inner: newCompactor(client, probePrompt, config.Compactor), slots: make(chan struct{}, builds), } @@ -115,30 +115,52 @@ func probe(ctx context.Context, args []string, out io.Writer) (err error) { return errors.Join(err, memory.Close()) } - run := probeRun{ + run := ProbeRun{ memory: memory, client: client, usagePath: usagePath, } appended, stop := run.Run(ctx, entries) - sizes, err := modelLines(memory) + sizes, err := probeModelLines(memory) err = errors.Join(err, memory.Close()) if err != nil { return err } - records, err := readUsageFile(usagePath) + records, err := probeReadUsage(usagePath) if err != nil { return err } - return writeProbeReport(out, appended, stop, records, sizes) + return probeWriteReport(out, appended, stop, records, sizes) } -// probeRun appends the messages of a probe run to its chat and stops the run -// by the rules of probeStop. -type probeRun struct { +// ProbeSlotSummarizer lets at most cap(slots) builds call inner at once. A +// build past the limit waits for a slot. +type ProbeSlotSummarizer struct { + inner Summarizer + slots chan struct{} +} + +// Summarize waits for a slot and builds the node with inner. +func (s ProbeSlotSummarizer) Summarize(ctx context.Context, task Task) (string, error) { + select { + case s.slots <- struct{}{}: + case <-ctx.Done(): + return "", ctx.Err() + } + + defer func() { + <-s.slots + }() + + return s.inner.Summarize(ctx, task) +} + +// ProbeRun appends the messages of a probe run to its chat and stops the run by +// the rules of probeStop. +type ProbeRun struct { memory *Memory client *ModelClient usagePath string @@ -149,7 +171,7 @@ type probeRun struct { // for all builds at the end. It returns the number of messages appended and why // the run stopped early, or "" when it did not. After a stop it waits for the // calls in flight, so no sent call is cancelled. -func (r *probeRun) Run(ctx context.Context, entries []probeEntry) (int, string) { +func (r *ProbeRun) Run(ctx context.Context, entries []ProbeEntry) (int, string) { appended, reason := r.append(ctx, entries) if reason == "" { return appended, "" @@ -164,7 +186,7 @@ func (r *probeRun) Run(ctx context.Context, entries []probeEntry) (int, string) } // append appends the entries and stops at the first stop rule. -func (r *probeRun) append(ctx context.Context, entries []probeEntry) (int, string) { +func (r *ProbeRun) append(ctx context.Context, entries []ProbeEntry) (int, string) { for n, entry := range entries { if entry.Kind == "user" { reason := r.wait(ctx) @@ -189,7 +211,7 @@ func (r *probeRun) append(ctx context.Context, entries []probeEntry) (int, strin // wait returns when no build runs, or when a stop rule ends the run. It checks // the rules each second, so a run stops soon also during a long wait. -func (r *probeRun) wait(ctx context.Context) string { +func (r *ProbeRun) wait(ctx context.Context) string { for { reason := r.check() if reason != "" { @@ -210,12 +232,12 @@ func (r *probeRun) wait(ctx context.Context) string { } // check returns why the run must stop, or "" when it can go on. -func (r *probeRun) check() string { +func (r *ProbeRun) check() string { if r.client.LimitReached() { return fmt.Sprintf("call limit of %d reached", r.client.limit) } - records, err := readUsageFile(r.usagePath) + records, err := probeReadUsage(r.usagePath) if err != nil { return err.Error() } @@ -223,42 +245,14 @@ func (r *probeRun) check() string { return probeStop(records) } -// slotSummarizer lets at most cap(slots) builds call inner at once. A build -// past the limit waits for a slot. -type slotSummarizer struct { - inner Summarizer - slots chan struct{} -} - -// Summarize waits for a slot and builds the node with inner. -func (s slotSummarizer) Summarize(ctx context.Context, task Task) (string, error) { - select { - case s.slots <- struct{}{}: - case <-ctx.Done(): - return "", ctx.Err() - } - - defer func() { - <-s.slots - }() - - return s.inner.Summarize(ctx, task) -} - -// probePrompt returns the system prompt of a probe run: the prompt of the agent -// probeName, without instructions. -func probePrompt() (string, error) { - return prompt(probeName, ""), nil -} - -// probeEntry is one message of a probe log. -type probeEntry struct { +// ProbeEntry is one message of a probe log. +type ProbeEntry struct { Kind string `json:"kind"` Text string `json:"text"` } -// readProbeLog reads the first count messages of the probe log at path. -func readProbeLog(path string, count int) (entries []probeEntry, err error) { +// probeReadLog reads the first count messages of the probe log at path. +func probeReadLog(path string, count int) (entries []ProbeEntry, err error) { file, err := os.Open(path) if err != nil { return nil, fmt.Errorf("open probe log: %w", err) @@ -271,7 +265,7 @@ func readProbeLog(path string, count int) (entries []probeEntry, err error) { scanner := bufio.NewScanner(file) scanner.Buffer(make([]byte, 64*1024), probeLineBytes) for len(entries) < count && scanner.Scan() { - var entry probeEntry + var entry ProbeEntry err = json.Unmarshal(scanner.Bytes(), &entry) if err != nil { return nil, fmt.Errorf("probe log line %d: %w", len(entries)+1, err) @@ -292,34 +286,15 @@ func readProbeLog(path string, count int) (entries []probeEntry, err error) { return entries, nil } -// probeStop returns why a probe run must stop: a failed call, or a quota window -// whose used percent grew by more than probeQuotaLimit points since the first -// call. It returns "" when the run can go on. -func probeStop(records []ModelUsageRecord) string { - for _, record := range records { - if record.Error != "" { - return "call failed: " + record.Error - } - } - - if len(records) == 0 { - return "" - } - - first := usedPercents(records[0]) - for name, used := range usedPercents(records[len(records)-1]) { - start, found := first[name] - if found && used-start > probeQuotaLimit { - return fmt.Sprintf("%s grew from %g to %g", name, start, used) - } - } - - return "" +// probePrompt returns the system prompt of a probe run: the prompt of the agent +// probeName, without instructions. +func probePrompt() (string, error) { + return prompt(probeName, ""), nil } -// modelLines returns the sizes of the nodes that the model wrote, in tree +// probeModelLines returns the sizes of the nodes that the model wrote, in tree // order. A node that fits its source with no model call is not one of them. -func modelLines(memory *Memory) ([]int, error) { +func probeModelLines(memory *Memory) ([]int, error) { memory.mutex.Lock() defer memory.mutex.Unlock() @@ -335,7 +310,7 @@ func modelLines(memory *Memory) ([]int, error) { return nil, err } - model, err := builtByModel(memory, key) + model, err := probeBuiltByModel(memory, key) if err != nil { return nil, err } @@ -348,34 +323,10 @@ func modelLines(memory *Memory) ([]int, error) { return sizes, nil } -// builtByModel tells if the node key needed a model call: a leaf whose source -// is over nodeBytes, or a merge whose children do not fit in one node. -func builtByModel(memory *Memory, key nodeKey) (bool, error) { - if key.level == 0 { - message, err := memory.log.Message(key.index) - if err != nil { - return false, err - } - - _, free := freeLeaf(message) - - return !free, nil - } - - left, right, err := memory.children(key) - if err != nil { - return false, err - } - - _, free := freeMerge(left, right) - - return !free, nil -} - -// readUsageFile reads the records of usage.jsonl at path, in the order of the +// probeReadUsage reads the records of usage.jsonl at path, in the order of the // start of their calls. The file has them in the order of the end of their // calls, which differs when calls run at once. -func readUsageFile(path string) (records []ModelUsageRecord, err error) { +func probeReadUsage(path string) (records []ModelUsageRecord, err error) { file, err := os.Open(path) if errors.Is(err, os.ErrNotExist) { return nil, nil @@ -407,43 +358,14 @@ func readUsageFile(path string) (records []ModelUsageRecord, err error) { } slices.SortStableFunc(records, func(a, b ModelUsageRecord) int { - return callStart(a).Compare(callStart(b)) + return probeCallStart(a).Compare(probeCallStart(b)) }) return records, nil } -// callStart returns the start of the call of record, or the zero time when its -// date is not valid. -func callStart(record ModelUsageRecord) time.Time { - start, err := time.Parse(time.RFC3339Nano, record.Date) - if err != nil { - return time.Time{} - } - - return start -} - -// usedPercents returns the used percent of each quota window of record, by -// header name. -func usedPercents(record ModelUsageRecord) map[string]float64 { - used := make(map[string]float64) - for name, value := range record.Quota { - if !strings.HasSuffix(name, "-used-percent") { - continue - } - - percent, err := strconv.ParseFloat(value, 64) - if err == nil { - used[name] = percent - } - } - - return used -} - -// writeProbeReport writes the result of a probe run. -func writeProbeReport(out io.Writer, appended int, stop string, records []ModelUsageRecord, sizes []int) error { +// probeWriteReport writes the result of a probe run. +func probeWriteReport(out io.Writer, appended int, stop string, records []ModelUsageRecord, sizes []int) error { var input, cached, output, failed int for _, record := range records { input += record.InputTokens @@ -461,23 +383,23 @@ func writeProbeReport(out io.Writer, appended int, stop string, records []ModelU lines = append(lines, fmt.Sprintf("messages: %d", appended), - fmt.Sprintf("calls: %d (%d failed), %.2f per message", len(records), failed, ratio(len(records), appended)), - fmt.Sprintf("model lines: %d, retries: %d, %.2f calls per model line", len(sizes), len(records)-failed-len(sizes), ratio(len(records), len(sizes))), - fmt.Sprintf("tokens: %d input, %d cached (%.1f%%), %d output", input, cached, 100*ratio(cached, input), output), + fmt.Sprintf("calls: %d (%d failed), %.2f per message", len(records), failed, probeRatio(len(records), appended)), + fmt.Sprintf("model lines: %d, retries: %d, %.2f calls per model line", len(sizes), len(records)-failed-len(sizes), probeRatio(len(records), len(sizes))), + fmt.Sprintf("tokens: %d input, %d cached (%.1f%%), %d output", input, cached, 100*probeRatio(cached, input), output), ) if len(sizes) > 0 { - lines = append(lines, sizeText(sizes)) + lines = append(lines, probeSizeText(sizes)) } if len(records) > 0 { - lines = append(lines, timeText(records)) + lines = append(lines, probeTimeText(records)) } if len(records) > 0 { lines = append(lines, - "quota at the first call: "+quotaText(records[0]), - "quota at the last call: "+quotaText(records[len(records)-1]), + "quota at the first call: "+probeQuotaText(records[0]), + "quota at the last call: "+probeQuotaText(records[len(records)-1]), ) } @@ -486,8 +408,77 @@ func writeProbeReport(out io.Writer, appended int, stop string, records []ModelU return err } -// sizeText describes the sizes of the model lines. -func sizeText(sizes []int) string { +// probeStop returns why a probe run must stop: a failed call, or a quota window +// whose used percent grew by more than probeQuotaLimit points since the first +// call. It returns "" when the run can go on. +func probeStop(records []ModelUsageRecord) string { + for _, record := range records { + if record.Error != "" { + return "call failed: " + record.Error + } + } + + if len(records) == 0 { + return "" + } + + first := probeUsedPercents(records[0]) + for name, used := range probeUsedPercents(records[len(records)-1]) { + start, found := first[name] + if found && used-start > probeQuotaLimit { + return fmt.Sprintf("%s grew from %g to %g", name, start, used) + } + } + + return "" +} + +// probeBuiltByModel tells if the node key needed a model call: a leaf whose +// source is over nodeBytes, or a merge whose children do not fit in one node. +func probeBuiltByModel(memory *Memory, key nodeKey) (bool, error) { + if key.level == 0 { + message, err := memory.log.Message(key.index) + if err != nil { + return false, err + } + + _, free := freeLeaf(message) + + return !free, nil + } + + left, right, err := memory.children(key) + if err != nil { + return false, err + } + + _, free := freeMerge(left, right) + + return !free, nil +} + +// probeCallStart returns the start of the call of record, or the zero time when +// its date is not valid. +func probeCallStart(record ModelUsageRecord) time.Time { + start, err := time.Parse(time.RFC3339Nano, record.Date) + if err != nil { + return time.Time{} + } + + return start +} + +// probeRatio returns a / b, or 0 when b is 0. +func probeRatio(a, b int) float64 { + if b == 0 { + return 0 + } + + return float64(a) / float64(b) +} + +// probeSizeText describes the sizes of the model lines. +func probeSizeText(sizes []int) string { sorted := slices.Sorted(slices.Values(sizes)) total, over := 0, 0 for _, size := range sorted { @@ -497,12 +488,12 @@ func sizeText(sizes []int) string { } } - return fmt.Sprintf("line bytes: min %d, median %d, mean %.0f, max %d, %d over %d", sorted[0], sorted[len(sorted)/2], ratio(total, len(sorted)), sorted[len(sorted)-1], over, nodeBytes) + return fmt.Sprintf("line bytes: min %d, median %d, mean %.0f, max %d, %d over %d", sorted[0], sorted[len(sorted)/2], probeRatio(total, len(sorted)), sorted[len(sorted)-1], over, nodeBytes) } -// timeText describes the time per call and how many calls ran at once. A call -// runs from its date for its seconds. -func timeText(records []ModelUsageRecord) string { +// probeTimeText describes the time per call and how many calls ran at once. A +// call runs from its date for its seconds. +func probeTimeText(records []ModelUsageRecord) string { type event struct { at time.Time delta int @@ -549,8 +540,8 @@ func timeText(records []ModelUsageRecord) string { return fmt.Sprintf("time: %.0f s in all, %.1f calls per minute, %.1f s per call (median %.1f, max %.1f), calls at once: at most %d, %.1f on average", wall, 60*float64(len(seconds))/max(wall, 1e-9), total/float64(len(seconds)), seconds[len(seconds)/2], seconds[len(seconds)-1], most, total/max(wall, 1e-9)) } -// quotaText returns the quota headers of record, sorted by name. -func quotaText(record ModelUsageRecord) string { +// probeQuotaText returns the quota headers of record, sorted by name. +func probeQuotaText(record ModelUsageRecord) string { names := slices.Sorted(maps.Keys(record.Quota)) parts := make([]string, len(names)) for n, name := range names { @@ -560,11 +551,20 @@ func quotaText(record ModelUsageRecord) string { return strings.Join(parts, " ") } -// ratio returns a / b, or 0 when b is 0. -func ratio(a, b int) float64 { - if b == 0 { - return 0 +// probeUsedPercents returns the used percent of each quota window of record, by +// header name. +func probeUsedPercents(record ModelUsageRecord) map[string]float64 { + used := make(map[string]float64) + for name, value := range record.Quota { + if !strings.HasSuffix(name, "-used-percent") { + continue + } + + percent, err := strconv.ParseFloat(value, 64) + if err == nil { + used[name] = percent + } } - return float64(a) / float64(b) + return used } diff --git a/probe_test.go b/probe_test.go index afc6e2f..265d49d 100644 --- a/probe_test.go +++ b/probe_test.go @@ -126,7 +126,7 @@ func TestProbeRunStopsWhenTheQuotaGrows(t *testing.T) { run := newTestProbeRun(t, server) appended, stop := run.Run(t.Context(), probeLog(probeMessages)) - sizes, err := modelLines(run.memory) + sizes, err := probeModelLines(run.memory) require.NoError(t, err) require.NoError(t, run.memory.Close()) @@ -134,12 +134,12 @@ func TestProbeRunStopsWhenTheQuotaGrows(t *testing.T) { require.Less(t, appended, probeMessages) require.NotEmpty(t, sizes) - records, err := readUsageFile(run.usagePath) + records, err := probeReadUsage(run.usagePath) require.NoError(t, err) require.Greater(t, len(records), int(probeQuotaLimit)) var report strings.Builder - require.NoError(t, writeProbeReport(&report, appended, stop, records, sizes)) + require.NoError(t, probeWriteReport(&report, appended, stop, records, sizes)) require.Contains(t, report.String(), "stopped early: ") require.Contains(t, report.String(), "cached (80.0%)") require.Contains(t, report.String(), "line bytes: min 400, median 400, mean 400, max 400, 0 over 512") @@ -152,9 +152,9 @@ func TestProbeRunStopsAtTheCallLimit(t *testing.T) { run := newTestProbeRun(t, server) run.client.LimitCalls(5) - entries := make([]probeEntry, 40) + entries := make([]ProbeEntry, 40) for n := range entries { - entries[n] = probeEntry{ + entries[n] = ProbeEntry{ Kind: "echo", Text: strings.Repeat("long output ", 100), } @@ -188,7 +188,7 @@ func TestProbeRunWaitsOnlyBeforeUserMessages(t *testing.T) { run := newTestProbeRun(t, server) long := strings.Repeat("long output ", 100) - entries := []probeEntry{ + entries := []ProbeEntry{ { Kind: "echo", Text: long, @@ -236,7 +236,7 @@ func TestProbeRunWaitsOnlyBeforeUserMessages(t *testing.T) { func TestSlotSummarizerLimitsTheBuildsAtOnce(t *testing.T) { inner := newFakeSummarizer() inner.gate = make(chan struct{}) - summarizer := slotSummarizer{ + summarizer := ProbeSlotSummarizer{ inner: inner, slots: make(chan struct{}, 2), } @@ -274,7 +274,7 @@ func TestReadUsageFileSortsByCallStart(t *testing.T) { })) } - records, err := readUsageFile(path) + records, err := probeReadUsage(path) require.NoError(t, err) require.Equal(t, []string{"2026-01-01T12:00:00.5Z", "2026-01-01T12:00:01.25Z", "2026-01-01T12:00:02Z"}, []string{records[0].Date, records[1].Date, records[2].Date}) } @@ -288,36 +288,16 @@ func TestTimeTextCountsTheCallsAtOnce(t *testing.T) { } } - text := timeText([]ModelUsageRecord{record(0, 10), record(2, 4), record(3, 2), record(10, 10)}) + text := probeTimeText([]ModelUsageRecord{record(0, 10), record(2, 4), record(3, 2), record(10, 10)}) require.Equal(t, "time: 20 s in all, 12.0 calls per minute, 6.5 s per call (median 10.0, max 10.0), calls at once: at most 3, 1.3 on average", text) } -// probeLog returns count messages of made-up work on a small bakery order -// system, with the mix of kinds and sizes of syntheticMessage. The same count -// always gives the same log. -func probeLog(count int) []probeEntry { - g := probeWriter{ - random: rand.New(rand.NewPCG(7, 11)), - } - - messages := make([]probeEntry, count) - for id := range messages { - kind, text := g.message() - messages[id] = probeEntry{ - Kind: kind, - Text: text, - } - } - - return messages -} - -// probeWriter writes readable made-up messages. -type probeWriter struct { +// ProbeWriter writes readable made-up messages. +type ProbeWriter struct { random *rand.Rand } -func (g *probeWriter) message() (string, string) { +func (g *ProbeWriter) message() (string, string) { roll := g.random.IntN(100) switch { case roll < 5: @@ -327,7 +307,7 @@ func (g *probeWriter) message() (string, string) { case roll < 12: return "note", g.fill(g.noteLine, 50+g.random.IntN(600)) case roll < 13: - return "work", "[" + g.pick(people) + "] report: " + g.fill(g.replyLine, 300+g.random.IntN(3000)) + return "work", "[" + g.pick(probePeople) + "] report: " + g.fill(g.replyLine, 300+g.random.IntN(3000)) case roll < 56: return "tool", g.toolCall(60 + g.random.IntN(400)) case roll < 57: @@ -338,7 +318,7 @@ func (g *probeWriter) message() (string, string) { } // fill joins lines of write until the text has about size bytes. -func (g *probeWriter) fill(write func() string, size int) string { +func (g *ProbeWriter) fill(write func() string, size int) string { var text strings.Builder for text.Len() < size { if text.Len() > 0 { @@ -351,52 +331,48 @@ func (g *probeWriter) fill(write func() string, size int) string { return text.String() } -func (g *probeWriter) userLine() string { +func (g *ProbeWriter) userLine() string { return g.pick([]string{ - fmt.Sprintf("Please check why the %s %s.", g.pick(things), g.pick(problems)), - fmt.Sprintf("Move the %s to %s before %s.", g.pick(things), g.pick(places), g.pick(days)), - fmt.Sprintf("Thanks, that works. Next, %s.", g.pick(tasks)), - fmt.Sprintf("No, keep the old %s. %s asked for it.", g.pick(things), g.pick(people)), - fmt.Sprintf("How many %s orders came in on %s?", g.pick(breads), g.pick(days)), + fmt.Sprintf("Please check why the %s %s.", g.pick(probeThings), g.pick(probeProblems)), + fmt.Sprintf("Move the %s to %s before %s.", g.pick(probeThings), g.pick(probePlaces), g.pick(probeDays)), + fmt.Sprintf("Thanks, that works. Next, %s.", g.pick(probeTasks)), + fmt.Sprintf("No, keep the old %s. %s asked for it.", g.pick(probeThings), g.pick(probePeople)), + fmt.Sprintf("How many %s orders came in on %s?", g.pick(probeBreads), g.pick(probeDays)), }) } -func (g *probeWriter) replyLine() string { +func (g *ProbeWriter) replyLine() string { return g.pick([]string{ - fmt.Sprintf("I looked at %s. The %s %s because %s.", g.pick(files), g.pick(things), g.pick(problems), g.pick(causes)), - fmt.Sprintf("I changed %s, and the tests pass now.", g.pick(changes)), - fmt.Sprintf("There were %d %s orders on %s, %d of them late.", 10+g.random.IntN(90), g.pick(breads), g.pick(days), g.random.IntN(10)), - fmt.Sprintf("Open question: should %s get %s?", g.pick(people), g.pick(tasks)), + fmt.Sprintf("I looked at %s. The %s %s because %s.", g.pick(probeFiles), g.pick(probeThings), g.pick(probeProblems), g.pick(probeCauses)), + fmt.Sprintf("I changed %s, and the tests pass now.", g.pick(probeChanges)), + fmt.Sprintf("There were %d %s orders on %s, %d of them late.", 10+g.random.IntN(90), g.pick(probeBreads), g.pick(probeDays), g.random.IntN(10)), + fmt.Sprintf("Open question: should %s get %s?", g.pick(probePeople), g.pick(probeTasks)), }) } -func (g *probeWriter) noteLine() string { +func (g *ProbeWriter) noteLine() string { return g.pick([]string{ - fmt.Sprintf("%s prefers %s.", g.pick(people), g.pick(preferences)), - fmt.Sprintf("The bakery opens at %d and closes at %d on %s.", 5+g.random.IntN(3), 17+g.random.IntN(4), g.pick(days)), + fmt.Sprintf("%s prefers %s.", g.pick(probePeople), g.pick(probePreferences)), + fmt.Sprintf("The bakery opens at %d and closes at %d on %s.", 5+g.random.IntN(3), 17+g.random.IntN(4), g.pick(probeDays)), }) } -func (g *probeWriter) outputLine() string { - return g.pick([]string{ - fmt.Sprintf("%02d:%02d order %d for %d %s, packed by %s.", 6+g.random.IntN(12), g.random.IntN(60), 1000+g.random.IntN(9000), 1+g.random.IntN(24), g.pick(breads), g.pick(people)), - fmt.Sprintf("%s:%d: %s %s.", g.pick(files), 1+g.random.IntN(400), g.pick(things), g.pick(problems)), - fmt.Sprintf("ok bakery/%s %d.%03ds", g.pick(modules), g.random.IntN(3), g.random.IntN(1000)), - }) +func (g *ProbeWriter) pick(words []string) string { + return words[g.random.IntN(len(words))] } // toolCall returns a tool name and its JSON input of about size bytes. -func (g *probeWriter) toolCall(size int) string { +func (g *ProbeWriter) toolCall(size int) string { name := g.pick([]string{"shell", "read", "write", "edit"}) var input map[string]string switch name { case "shell": - input = map[string]string{"command": "grep -n '" + g.pick(things) + "' " + g.pick(files)} + input = map[string]string{"command": "grep -n '" + g.pick(probeThings) + "' " + g.pick(probeFiles)} case "read": - input = map[string]string{"path": g.pick(files)} + input = map[string]string{"path": g.pick(probeFiles)} default: input = map[string]string{ - "path": g.pick(files), + "path": g.pick(probeFiles), "text": g.fill(g.replyLine, max(0, size-80)), } } @@ -409,11 +385,35 @@ func (g *probeWriter) toolCall(size int) string { return name + " " + string(data) } -func (g *probeWriter) pick(words []string) string { - return words[g.random.IntN(len(words))] +func (g *ProbeWriter) outputLine() string { + return g.pick([]string{ + fmt.Sprintf("%02d:%02d order %d for %d %s, packed by %s.", 6+g.random.IntN(12), g.random.IntN(60), 1000+g.random.IntN(9000), 1+g.random.IntN(24), g.pick(probeBreads), g.pick(probePeople)), + fmt.Sprintf("%s:%d: %s %s.", g.pick(probeFiles), 1+g.random.IntN(400), g.pick(probeThings), g.pick(probeProblems)), + fmt.Sprintf("ok bakery/%s %d.%03ds", g.pick(probeModules), g.random.IntN(3), g.random.IntN(1000)), + }) +} + +// probeLog returns count messages of made-up work on a small bakery order +// system, with the mix of kinds and sizes of syntheticMessage. The same count +// always gives the same log. +func probeLog(count int) []ProbeEntry { + g := ProbeWriter{ + random: rand.New(rand.NewPCG(7, 11)), + } + + messages := make([]ProbeEntry, count) + for id := range messages { + kind, text := g.message() + messages[id] = ProbeEntry{ + Kind: kind, + Text: text, + } + } + + return messages } -func newTestProbeRun(t *testing.T, server *ModelFake) *probeRun { +func newTestProbeRun(t *testing.T, server *ModelFake) *ProbeRun { t.Helper() root := t.TempDir() @@ -422,7 +422,7 @@ func newTestProbeRun(t *testing.T, server *ModelFake) *probeRun { memory, err := openMemory(t.Context(), root, newTestClock().Now, newCompactor(client, probePrompt, defaultConfig().Compactor)) require.NoError(t, err) - return &probeRun{ + return &ProbeRun{ memory: memory, client: client, usagePath: usagePath, @@ -430,16 +430,16 @@ func newTestProbeRun(t *testing.T, server *ModelFake) *probeRun { } var ( - people = []string{"Mara", "Joel", "Ines", "Tomas", "Priya", "Lena"} - breads = []string{"rye", "sourdough", "baguette", "brioche", "spelt", "seed loaf"} - days = []string{"Monday", "Tuesday", "Wednesday", "Thursday", "Friday", "Saturday"} - things = []string{"order form", "price list", "delivery route", "stock count", "invoice export", "oven schedule"} - problems = []string{"shows the wrong total", "skips the last row", "is slow at noon", "loses the customer note", "rounds the tax down"} - places = []string{"the front page", "the weekly report", "the shared folder", "the admin menu"} - tasks = []string{"add a field for allergies", "send the invoices on Friday", "split the route in two", "print labels for the shelf"} - causes = []string{"the date has no time zone", "the cache keeps old prices", "the loop stops one row early", "the form trims the note"} - changes = []string{"the total to use cents", "the route to sort by street", "the export to add the tax line", "the form to keep the note"} - preferences = []string{"short answers", "a call before a change to prices", "invoices as PDF", "reports in the morning"} - files = []string{"orders/form.go", "prices/list.go", "routes/plan.go", "stock/count.go", "export/invoice.go", "ovens/schedule.go"} - modules = []string{"orders", "prices", "routes", "stock", "export", "ovens"} + probePeople = []string{"Mara", "Joel", "Ines", "Tomas", "Priya", "Lena"} + probeBreads = []string{"rye", "sourdough", "baguette", "brioche", "spelt", "seed loaf"} + probeDays = []string{"Monday", "Tuesday", "Wednesday", "Thursday", "Friday", "Saturday"} + probeThings = []string{"order form", "price list", "delivery route", "stock count", "invoice export", "oven schedule"} + probeProblems = []string{"shows the wrong total", "skips the last row", "is slow at noon", "loses the customer note", "rounds the tax down"} + probePlaces = []string{"the front page", "the weekly report", "the shared folder", "the admin menu"} + probeTasks = []string{"add a field for allergies", "send the invoices on Friday", "split the route in two", "print labels for the shelf"} + probeCauses = []string{"the date has no time zone", "the cache keeps old prices", "the loop stops one row early", "the form trims the note"} + probeChanges = []string{"the total to use cents", "the route to sort by street", "the export to add the tax line", "the form to keep the note"} + probePreferences = []string{"short answers", "a call before a change to prices", "invoices as PDF", "reports in the morning"} + probeFiles = []string{"orders/form.go", "prices/list.go", "routes/plan.go", "stock/count.go", "export/invoice.go", "ovens/schedule.go"} + probeModules = []string{"orders", "prices", "routes", "stock", "export", "ovens"} )