-
-// probeModelLines returns the sizes of the nodes that the model wrote, in tree
-// order. A node that fits its source with no model call is not one of them.
-func probeModelLines(memory *Memory) ([]int, error) {
- memory.mutex.Lock()
- defer memory.mutex.Unlock()
-
- keys, err := memory.tree.Keys()
- if err != nil {
- return nil, err
- }
-
- var sizes []int
- for _, key := range keys {
- node, _, err := memory.tree.Node(key.level, key.index)
- if err != nil {
- return nil, err
- }
-
- model, err := probeBuiltByModel(memory, key)
- if err != nil {
- return nil, err
- }
-
- if model {
- sizes = append(sizes, node.Size)
- }
- }
-
- return sizes, nil
-}
-
-// probeReadUsage reads the records of usage.jsonl at path, in the order of the
-// start of their calls. The file has them in the order of the end of their
-// calls, which differs when calls run at once.
-func probeReadUsage(path string) (records []ModelUsageRecord, err error) {
- file, err := os.Open(path)
- if errors.Is(err, os.ErrNotExist) {
- return nil, nil
- }
-
- if err != nil {
- return nil, fmt.Errorf("open usage file: %w", err)
- }
-
- defer func() {
- err = errors.Join(err, file.Close())
- }()
-
- scanner := bufio.NewScanner(file)
- scanner.Buffer(make([]byte, 64*1024), probeLineBytes)
- for scanner.Scan() {
- var record ModelUsageRecord
- err = json.Unmarshal(scanner.Bytes(), &record)
- if err != nil {
- return nil, fmt.Errorf("decode usage: %w", err)
- }
-
- records = append(records, record)
- }
-
- err = scanner.Err()
- if err != nil {
- return nil, fmt.Errorf("read usage file: %w", err)
- }
-
- slices.SortStableFunc(records, func(a, b ModelUsageRecord) int {
- return probeCallStart(a).Compare(probeCallStart(b))
- })
-
- return records, nil
-}
-
-// probeWriteReport writes the result of a probe run.
-func probeWriteReport(out io.Writer, appended int, stop string, records []ModelUsageRecord, sizes []int) error {
- var input, cached, output, failed int
- for _, record := range records {
- input += record.InputTokens
- cached += record.CachedTokens
- output += record.OutputTokens
- if record.Error != "" {
- failed++
- }
- }
-
- var lines []string
- if stop != "" {
- lines = append(lines, "stopped early: "+stop)
- }
-
- lines = append(lines,
- fmt.Sprintf("messages: %d", appended),
- fmt.Sprintf("calls: %d (%d failed), %.2f per message", len(records), failed, probeRatio(len(records), appended)),
- fmt.Sprintf("model lines: %d, retries: %d, %.2f calls per model line", len(sizes), len(records)-failed-len(sizes), probeRatio(len(records), len(sizes))),
- fmt.Sprintf("tokens: %d input, %d cached (%.1f%%), %d output", input, cached, 100*probeRatio(cached, input), output),
- )
-
- if len(sizes) > 0 {
- lines = append(lines, probeSizeText(sizes))
- }
-
- if len(records) > 0 {
- lines = append(lines, probeTimeText(records))
- }
-
- if len(records) > 0 {
- lines = append(lines,
- "quota at the first call: "+probeQuotaText(records[0]),
- "quota at the last call: "+probeQuotaText(records[len(records)-1]),
- )
- }
-
- _, err := io.WriteString(out, strings.Join(lines, "\n")+"\n")
-
- return err
-}
-
-// probeStop returns why a probe run must stop: a failed call, or a quota window
-// whose used percent grew by more than probeQuotaLimit points since the first
-// call. It returns "" when the run can go on.
-func probeStop(records []ModelUsageRecord) string {
- for _, record := range records {
- if record.Error != "" {
- return "call failed: " + record.Error
- }
- }
-
- if len(records) == 0 {
- return ""
- }
-
- first := probeUsedPercents(records[0])
- for name, used := range probeUsedPercents(records[len(records)-1]) {
- start, found := first[name]
- if found && used-start > probeQuotaLimit {
- return fmt.Sprintf("%s grew from %g to %g", name, start, used)
- }
- }
-
- return ""
-}
-
-// probeBuiltByModel tells if the node key needed a model call: a leaf whose
-// source is over nodeBytes, or a merge whose children do not fit in one node.
-func probeBuiltByModel(memory *Memory, key NodeKey) (bool, error) {
- if key.level == 0 {
- message, err := memory.log.Message(key.index)
- if err != nil {
- return false, err
- }
-
- _, free := freeLeaf(message)
-
- return !free, nil
- }
-
- left, right, err := memory.children(key)
- if err != nil {
- return false, err
- }
-
- _, free := freeMerge(left, right)
-
- return !free, nil
-}
-
-// probeCallStart returns the start of the call of record, or the zero time when
-// its date is not valid.
-func probeCallStart(record ModelUsageRecord) time.Time {
- start, err := time.Parse(time.RFC3339Nano, record.Date)
- if err != nil {
- return time.Time{}
- }
-
- return start
-}
-
-// probeRatio returns a / b, or 0 when b is 0.
-func probeRatio(a, b int) float64 {
- if b == 0 {
- return 0
- }
-
- return float64(a) / float64(b)
-}
-
-// probeSizeText describes the sizes of the model lines.
-func probeSizeText(sizes []int) string {
- sorted := slices.Sorted(slices.Values(sizes))
- total, over := 0, 0
- for _, size := range sorted {
- total += size
- if size > nodeBytes {
- over++
- }
- }
-
- return fmt.Sprintf("line bytes: min %d, median %d, mean %.0f, max %d, %d over %d", sorted[0], sorted[len(sorted)/2], probeRatio(total, len(sorted)), sorted[len(sorted)-1], over, nodeBytes)
-}
-
-// probeTimeText describes the time per call and how many calls ran at once. A
-// call runs from its date for its seconds.
-func probeTimeText(records []ModelUsageRecord) string {
- type event struct {
- at time.Time
- delta int
- }
-
- var events []event
- seconds := make([]float64, 0, len(records))
- total := 0.0
- for _, record := range records {
- start, err := time.Parse(time.RFC3339Nano, record.Date)
- if err != nil {
- continue
- }
-
- length := time.Duration(record.Seconds * float64(time.Second))
- events = append(events, event{
- at: start,
- delta: 1,
- }, event{
- at: start.Add(length),
- delta: -1,
- })
- seconds = append(seconds, record.Seconds)
- total += record.Seconds
- }
-
- if len(events) == 0 {
- return "time: no dated calls"
- }
-
- slices.SortFunc(events, func(a, b event) int {
- return cmp.Or(a.at.Compare(b.at), cmp.Compare(a.delta, b.delta))
- })
-
- running, most := 0, 0
- for _, e := range events {
- running += e.delta
- most = max(most, running)
- }
-
- wall := events[len(events)-1].at.Sub(events[0].at).Seconds()
- slices.Sort(seconds)
-
- return fmt.Sprintf("time: %.0f s in all, %.1f calls per minute, %.1f s per call (median %.1f, max %.1f), calls at once: at most %d, %.1f on average", wall, 60*float64(len(seconds))/max(wall, 1e-9), total/float64(len(seconds)), seconds[len(seconds)/2], seconds[len(seconds)-1], most, total/max(wall, 1e-9))
-}
-
-// probeQuotaText returns the quota headers of record, sorted by name.
-func probeQuotaText(record ModelUsageRecord) string {
- names := slices.Sorted(maps.Keys(record.Quota))
- parts := make([]string, len(names))
- for n, name := range names {
- parts[n] = strings.TrimPrefix(name, "x-codex-") + "=" + record.Quota[name]
- }
-
- return strings.Join(parts, " ")
-}
-
-// probeUsedPercents returns the used percent of each quota window of record, by
-// header name.
-func probeUsedPercents(record ModelUsageRecord) map[string]float64 {
- used := make(map[string]float64)
- for name, value := range record.Quota {
- if !strings.HasSuffix(name, "-used-percent") {
- continue
- }
-
- percent, err := strconv.ParseFloat(value, 64)
- if err == nil {
- used[name] = percent
- }
- }
-
- return used
-}
projects / onechat / blobdiff