Write each line to the OS without an fsync

Each log line and each node goes to the operating system with one write
and no buffer before the call returns, so a crash of the process loses
nothing. view.json stays atomic through a temporary file and a rename.
An fsync on each write cost about 96 ms per message on a typical disk,
which made a long replay take hours. Durability against a power loss is
a later group commit, with one background sync that makes all earlier
writes durable.

Issue: beb9a70

diff --git a/journal.go b/journal.go
index 3c2a7ef52e4e24cbd476f0705e5771739312c470..a962011785fc4f665bfb5e67bb5403b8b5d037e5 100644 (file)
--- a/journal.go
+++ b/journal.go
@@ -18,7 +18,8 @@ type journal struct {
        day  string
 }
 
        day  string
 }
 

-// Write appends line to the file of day and flushes it to the disk.
+// Write appends line to the file of day with one write to the operating system,
+// without a buffer. A crash of the process then loses nothing.

 func (j *journal) Write(day string, line []byte) error {
        if day != j.day {
                err := j.Close()
 func (j *journal) Write(day string, line []byte) error {
        if day != j.day {
                err := j.Close()


@@ -39,11 +40,6 @@ func (j *journal) Write(day string, line []byte) error {
                return fmt.Errorf("write line: %w", err)
        }
 
                return fmt.Errorf("write line: %w", err)
        }
 

-       err = j.file.Sync()
-       if err != nil {
-               return fmt.Errorf("flush line: %w", err)
-       }
-

        return nil
 }
 
        return nil
 }
 

diff --git a/log.go b/log.go
index 7b8d5126ec2659db6b9598d245a86b1f5acb5b9b..f1727d5fafef4e3dc35c42df94c9986f0b515aec 100644 (file)
--- a/log.go
+++ b/log.go
@@ -38,11 +38,11 @@ type Message struct {
 
 // Log is the append-only message log of one chat. It writes one JSON line for
 // each message to main/YYYY-MM-DD.jsonl, named for the local day of the write.
 
 // Log is the append-only message log of one chat. It writes one JSON line for
 // each message to main/YYYY-MM-DD.jsonl, named for the local day of the write.

-// It never edits or deletes a line. Each write goes to the file and to the disk
-// before Append returns. An exclusive lock on the lock file makes sure that
-// only one process writes the log. After a failed write, the end of the file is
-// not known, so the log refuses all writes until it is opened again. The caller
-// serializes all calls.
+// It never edits or deletes a line. Each line goes to the operating system
+// before Append returns, so a crash of the process loses no message. An
+// exclusive lock on the lock file makes sure that only one process writes the
+// log. After a failed write, the end of the file is not known, so the log
+// refuses all writes until it is opened again. The caller serializes all calls.

 type Log struct {
        clock    func() time.Time
        lock     *os.File
 type Log struct {
        clock    func() time.Time
        lock     *os.File

diff --git a/memory.go b/memory.go
index 9c8073f9b06fad474965ec2b9fb78812e0c9e2c9..9e38916b3a7aff1c0fd98940d5caef28c4c5b3c8 100644 (file)
--- a/memory.go
+++ b/memory.go
@@ -445,20 +445,16 @@ func (m *Memory) context(level, index int) string {
        return m.cview.Context(m.tree, end)
 }
 
        return m.cview.Context(m.tree, end)
 }
 

-// writeFileAtomic replaces the file at path with data. A reader or a crash sees
-// the old file or the new file, never a part of one.
+// writeFileAtomic replaces the file at path with data through a temporary file
+// and a rename. A reader or a crash of the process sees the old file or the new
+// file, never a part of one.

 func writeFileAtomic(path string, data []byte) error {
 func writeFileAtomic(path string, data []byte) error {

-       dir := filepath.Dir(path)
-       file, err := os.CreateTemp(dir, "."+filepath.Base(path)+".*")
+       file, err := os.CreateTemp(filepath.Dir(path), "."+filepath.Base(path)+".*")

        if err != nil {
                return fmt.Errorf("create temporary file: %w", err)
        }
 
        _, err = file.Write(data)
        if err != nil {
                return fmt.Errorf("create temporary file: %w", err)
        }
 
        _, err = file.Write(data)

-       if err == nil {
-               err = file.Sync()
-       }
-

        err = errors.Join(err, file.Close())
        if err == nil {
                err = os.Rename(file.Name(), path)
        err = errors.Join(err, file.Close())
        if err == nil {
                err = os.Rename(file.Name(), path)


@@ -468,24 +464,5 @@ func writeFileAtomic(path string, data []byte) error {
                return errors.Join(fmt.Errorf("replace file: %w", err), os.Remove(file.Name()))
        }
 
                return errors.Join(fmt.Errorf("replace file: %w", err), os.Remove(file.Name()))
        }
 

-       return syncDir(dir)
-}
-
-// syncDir flushes the entries of dir to the disk.
-func syncDir(dir string) (err error) {
-       handle, err := os.Open(dir)
-       if err != nil {
-               return fmt.Errorf("open directory: %w", err)
-       }
-
-       defer func() {
-               err = errors.Join(err, handle.Close())
-       }()
-
-       err = handle.Sync()
-       if err != nil {
-               return fmt.Errorf("flush directory: %w", err)
-       }
-

        return nil
 }
        return nil
 }