package main import ( "context" "encoding/json" "errors" "fmt" "os" "path/filepath" "time" "unicode/utf8" ) // logKinds holds the logKinds that a message can have. var logKinds = map[string]bool{ "user": true, "reply": true, "tool": true, "echo": true, "work": true, "note": true, } // openLog locks the chat in root and indexes its new lines. The clock gives the // date of each new message. func openLog(ctx context.Context, root string, clock func() time.Time) (*Log, error) { for _, dir := range []string{"main", "index"} { err := os.MkdirAll(filepath.Join(root, dir), 0o700) if err != nil { return nil, fmt.Errorf("create %s directory: %w", dir, err) } } lock, err := lockFile(filepath.Join(root, "lock")) if err != nil { return nil, err } l := &Log{ clock: clock, lock: lock, journal: Journal{ dir: filepath.Join(root, "main"), }, } l.index, err = openRecords(filepath.Join(root, "index", "main.idx"), positionBytes) if err != nil { return nil, errors.Join(err, l.Close()) } err = l.catchUp(ctx) if err != nil { return nil, errors.Join(fmt.Errorf("index message log: %w", err), l.Close()) } return l, nil } // Log is the append-only message log of one chat. It writes one JSON line for // each message to main/YYYY-MM-DD.jsonl, named for the local day of the write. // It never edits or deletes a line. Each line goes to the operating system // before Append returns, so a crash of the process loses no message. An // exclusive lock on the lock file makes sure that only one process writes the // log. After a failed write, the end of the file is not known, so the log // refuses all writes until it is opened again. The caller serializes all calls. // // The log keeps no message in memory. The index index/main.idx holds the // position of message i in record i, so a read of a message is one read of the // index and one of its day file. At open the log indexes the lines after the // last indexed one, so a missing index is built again from the day files. type Log struct { clock func() time.Time lock *os.File journal Journal index *Records failure error } // Close releases the lock. The log refuses all calls after Close. func (l *Log) Close() error { err := l.journal.Close() if l.index != nil { err = errors.Join(err, l.index.Close()) l.index = nil } if l.lock != nil { err = errors.Join(err, l.lock.Close()) l.lock = nil } return err } // Append writes a new message to the log and returns it with its ID, size, and // date. func (l *Log) Append(kind, text string) (Message, error) { if l.lock == nil { return Message{}, os.ErrClosed } if l.failure != nil { return Message{}, fmt.Errorf("log needs a reopen after a failed write: %w", l.failure) } now := l.clock() message := Message{ ID: l.index.count, Kind: kind, Text: text, Size: len(text), Date: now.Format(time.RFC3339Nano), } err := logValidateMessage(message) if err != nil { return Message{}, err } line, err := json.Marshal(message) if err != nil { return Message{}, fmt.Errorf("encode message: %w", err) } place, err := l.journal.Write(now.Format(time.DateOnly), append(line, '\n')) if err == nil { err = l.addIndex(place) } if err != nil { l.failure = err return Message{}, fmt.Errorf("write message %d: %w", message.ID, err) } return message, nil } // Len returns the number of messages in the log. func (l *Log) Len() int { return l.index.count } // Message reads the message with the given ID from its day file. func (l *Log) Message(id int) (Message, error) { record, found, err := l.index.Read(id) if err != nil { return Message{}, err } if !found { return Message{}, fmt.Errorf("no message %d", id) } line, err := l.journal.Read(decodePosition(record)) if err != nil { return Message{}, fmt.Errorf("read message %d: %w", id, err) } var message Message err = json.Unmarshal(line, &message) if err != nil { return Message{}, fmt.Errorf("decode message %d: %w", id, err) } if message.ID != id { return Message{}, fmt.Errorf("index of message %d points to message %d", id, message.ID) } return message, nil } // catchUp indexes the lines after the last indexed line. func (l *Log) catchUp(ctx context.Context) error { start := Position{} if l.index.count > 0 { record, _, err := l.index.Read(l.index.count - 1) if err != nil { return err } last := decodePosition(record) start = Position{ day: last.day, offset: last.offset + uint64(last.length), } } return l.journal.Scan(ctx, start, func(line []byte, place Position) error { var message Message err := json.Unmarshal(line, &message) if err != nil { return err } if message.ID != l.index.count { return fmt.Errorf("expected message %d, found %d", l.index.count, message.ID) } err = logValidateMessage(message) if err != nil { return fmt.Errorf("message %d: %w", message.ID, err) } return l.addIndex(place) }) } func (l *Log) addIndex(place Position) error { record := make([]byte, positionBytes) encodePosition(record, place) return l.index.Append(record) } // Message is one entry of the chat log. ID is the permanent position of the // message in the chat, from 0. Size is the UTF-8 byte length of Text. Date is // the time of the write in RFC 3339 format. type Message struct { ID int `json:"i"` Kind string `json:"kind"` Text string `json:"text"` Size int `json:"size"` Date string `json:"date"` } // logValidateMessage makes sure that a message has a known kind, valid UTF-8 // text, the size of its text, and a valid date. func logValidateMessage(message Message) error { if !logKinds[message.Kind] { return fmt.Errorf("unknown message kind %q", message.Kind) } if !utf8.ValidString(message.Text) { return errors.New("message text is not valid UTF-8") } if message.Size != len(message.Text) { return fmt.Errorf("message size %d is not the text size %d", message.Size, len(message.Text)) } _, err := time.Parse(time.RFC3339Nano, message.Date) if err != nil { return fmt.Errorf("parse message date: %w", err) } return nil }