package main import ( "bufio" "context" "encoding/json" "errors" "fmt" "os" "path/filepath" ) // journal appends JSON lines to one file for each day in dir, named // YYYY-MM-DD.jsonl. It keeps the file of the last day open. type journal struct { dir string file *os.File day string } // Write appends line to the file of day with one write to the operating system, // without a buffer. A crash of the process then loses nothing. func (j *journal) Write(day string, line []byte) error { if day != j.day { err := j.Close() if err != nil { return err } j.file, err = os.OpenFile(filepath.Join(j.dir, day+".jsonl"), os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0o600) if err != nil { return fmt.Errorf("open day file: %w", err) } j.day = day } _, err := j.file.Write(line) if err != nil { return fmt.Errorf("write line: %w", err) } return nil } // Close closes the open day file. func (j *journal) Close() error { if j.file == nil { return nil } err := j.file.Close() j.file = nil j.day = "" if err != nil { return fmt.Errorf("close day file: %w", err) } return nil } // readLines decodes each line of the day files in dir, oldest day first, and // gives it to add in file order. func readLines[T any](ctx context.Context, dir string, add func(T) error) error { paths, err := filepath.Glob(filepath.Join(dir, "*.jsonl")) if err != nil { return fmt.Errorf("list day files: %w", err) } for _, path := range paths { err = readFile(ctx, path, add) if err != nil { return err } } return nil } func readFile[T any](ctx context.Context, path string, add func(T) error) (err error) { file, err := os.Open(path) if err != nil { return fmt.Errorf("open day file: %w", err) } defer func() { err = errors.Join(err, file.Close()) }() scanner := bufio.NewScanner(file) scanner.Buffer(nil, 1<<30) for number := 1; scanner.Scan(); number++ { if ctx.Err() != nil { return ctx.Err() } var value T err = json.Unmarshal(scanner.Bytes(), &value) if err == nil { err = add(value) } if err != nil { return fmt.Errorf("%s line %d: %w", filepath.Base(path), number, err) } } err = scanner.Err() if err != nil { return fmt.Errorf("read %s: %w", filepath.Base(path), err) } return nil }