Use both logs for WhatsApp import cutoffs

[onechat] / whatsapp_import.go

diff --git a/whatsapp_import.go b/whatsapp_import.go
index e1cbd029522810c832a35ff81e676d8fdd05a7bb..23619157ac437de4074b1b8b1f87eefca9cf3513 100644 (file)
--- a/whatsapp_import.go
+++ b/whatsapp_import.go
@@ -87,7 +87,7 @@ func whatsappImport(ctx context.Context, args []string, out io.Writer) (err erro
                return fmt.Errorf("read import names: %w", err)
        }
 
                return fmt.Errorf("read import names: %w", err)
        }
 

-       plan, err := whatsappImportPlan(ctx, args, old, names, owner, time.Local)
+       plan, err := whatsappImportPlan(ctx, args, old, nil, names, owner, time.Local)

        if err != nil {
                return fmt.Errorf("plan import: %w", err)
        }
        if err != nil {
                return fmt.Errorf("plan import: %w", err)
        }


@@ -159,6 +159,7 @@ func whatsappImport(ctx context.Context, args []string, out io.Writer) (err erro
 // log. Its estimate uses the measured cost of 1.3 calls per message, not a
 // promise about a particular export or compaction retry.
 type WhatsAppImportPlan struct {
 // log. Its estimate uses the measured cost of 1.3 calls per message, not a
 // promise about a particular export or compaction retry.
 type WhatsAppImportPlan struct {

+       cutoffs    map[string]time.Time

        added      []DatedMessage
        chats      []string
        cutoff     time.Time
        added      []DatedMessage
        chats      []string
        cutoff     time.Time


@@ -193,7 +194,13 @@ func (p WhatsAppImportPlan) Write(out io.Writer) error {
        }
 
        for _, chat := range p.chats {
        }
 
        for _, chat := range p.chats {

-               _, err = fmt.Fprintf(out, "  %s\n", chat)
+               line := "  " + chat
+               cutoff, found := p.cutoffs[chat]
+               if found {
+                       line += " (cutoff: " + cutoff.Format(time.RFC3339) + ")"
+               }
+
+               _, err = fmt.Fprintln(out, line)

                if err != nil {
                        return fmt.Errorf("write import plan: %w", err)
                }
                if err != nil {
                        return fmt.Errorf("write import plan: %w", err)
                }


@@ -319,64 +326,24 @@ func whatsappImportName(names map[string]WhatsAppChat, chat WhatsAppChat) {
        names[chat.Name] = chat
 }
 
        names[chat.Name] = chat
 }
 

-// whatsappImportPlan filters exports at the first message of the earliest live
-// batch and skips chat-days already imported. Each remaining chat-day becomes a
-// work message dated at its last exported message. Overlapping chat-days in the
-// inputs are refused, not silently combined or duplicated.
-func whatsappImportPlan(ctx context.Context, paths []string, old []DatedMessage, names map[string]WhatsAppChat, owner string, location *time.Location) (WhatsAppImportPlan, error) {
-       plan := WhatsAppImportPlan{ownerKnown: owner != ""}
+// whatsappImportPlan filters each chat at its earliest live message across the
+// target and root logs. Only imported days in the target are skipped. Each
+// remaining chat-day becomes a work message dated at its last exported message.
+// Overlapping input chat-days are refused.
+func whatsappImportPlan(ctx context.Context, paths []string, old, rootLog []DatedMessage, names map[string]WhatsAppChat, owner string, location *time.Location) (WhatsAppImportPlan, error) {
+       plan := WhatsAppImportPlan{
+               ownerKnown: owner != "",
+               cutoffs:    make(map[string]time.Time),
+       }
+

        names = maps.Clone(names)
        if names == nil {
                names = make(map[string]WhatsAppChat)
        }
 
        names = maps.Clone(names)
        if names == nil {
                names = make(map[string]WhatsAppChat)
        }
 

-       imported := make(map[string]bool)
-       for _, entry := range old {
-               if entry.Kind != "work" || !strings.HasPrefix(entry.Text, "[WhatsApp] ") {
-                       continue
-               }
-
-               first, _, _ := strings.Cut(entry.Text, "\n")
-               marked := strings.HasSuffix(first, whatsappImportMark)
-               first = strings.TrimSuffix(first, whatsappImportMark)
-               match := whatsappImportHeader.FindStringSubmatch(first)
-               name, day := "", ""
-               if len(match) > 0 {
-                       name, day = match[1], match[3]
-                       whatsappImportName(names, WhatsAppChat{
-                               ID:    match[2],
-                               Name:  name,
-                               Group: strings.Contains(first, ", group "+match[2]+","),
-                       })
-               } else if marked {
-                       match = whatsappImportUnknown.FindStringSubmatch(first)
-                       if len(match) > 0 {
-                               name, day = match[1], match[2]
-                       }
-               }
-
-               if name == "" {
-                       return plan, errors.New("read WhatsApp history: invalid batch header")
-               }
-
-               if marked {
-                       imported[name+"\n"+day] = true
-                       continue
-               }
-
-               _, body, found := strings.Cut(entry.Text, "\n")
-               if !found || len(body) < len("15:04 ") {
-                       return plan, errors.New("read WhatsApp history: missing first message time")
-               }
-
-               date, err := time.ParseInLocation("2006-01-02 15:04", day+" "+body[:5], location)
-               if err != nil {
-                       return plan, fmt.Errorf("parse first live message time: %w", err)
-               }
-
-               if plan.cutoff.IsZero() || date.Before(plan.cutoff) {
-                       plan.cutoff = date
-               }
+       imported, err := whatsappImportHistory(old, rootLog, names, plan.cutoffs, location)
+       if err != nil {
+               return plan, fmt.Errorf("read import history: %w", err)

        }
 
        seen := make(map[string]bool)
        }
 
        seen := make(map[string]bool)


@@ -399,9 +366,14 @@ func whatsappImportPlan(ctx context.Context, paths []string, old []DatedMessage,
                        plan.chats = append(plan.chats, chat.Name)
                }
 
                        plan.chats = append(plan.chats, chat.Name)
                }
 

+               cutoff := plan.cutoffs[chat.Name]
+               if !cutoff.IsZero() && (plan.cutoff.IsZero() || cutoff.Before(plan.cutoff)) {
+                       plan.cutoff = cutoff
+               }
+

                days := make(map[string][]WhatsAppMessage)
                for _, message := range messages {
                days := make(map[string][]WhatsAppMessage)
                for _, message := range messages {

-                       if !plan.cutoff.IsZero() && !message.Time.Before(plan.cutoff) {
+                       if !cutoff.IsZero() && !message.Time.Before(cutoff) {

                                continue
                        }
 
                                continue
                        }
 


@@ -451,8 +423,6 @@ func whatsappImportPlan(ctx context.Context, paths []string, old []DatedMessage,
                }
        }
 
                }
        }
 

-       var err error
-

        plan.added, err = rebuildMerge(nil, plan.added)
        if err != nil {
                return plan, fmt.Errorf("order import batches: %w", err)
        plan.added, err = rebuildMerge(nil, plan.added)
        if err != nil {
                return plan, fmt.Errorf("order import batches: %w", err)


@@ -464,6 +434,66 @@ func whatsappImportPlan(ctx context.Context, paths []string, old []DatedMessage,
        return plan, nil
 }
 
        return plan, nil
 }
 

+// whatsappImportHistory reads batch names and live cutoffs from both logs.
+// Imported chat-days belong only to the target log, which is first.
+func whatsappImportHistory(target, root []DatedMessage, names map[string]WhatsAppChat, cutoffs map[string]time.Time, location *time.Location) (map[string]bool, error) {
+       imported := make(map[string]bool)
+       for source, entries := range [][]DatedMessage{target, root} {
+               for _, entry := range entries {
+                       if entry.Kind != "work" || !strings.HasPrefix(entry.Text, "[WhatsApp] ") {
+                               continue
+                       }
+
+                       first, _, _ := strings.Cut(entry.Text, "\n")
+                       marked := strings.HasSuffix(first, whatsappImportMark)
+                       first = strings.TrimSuffix(first, whatsappImportMark)
+                       match := whatsappImportHeader.FindStringSubmatch(first)
+                       name, day := "", ""
+                       if len(match) > 0 {
+                               name, day = match[1], match[3]
+                               whatsappImportName(names, WhatsAppChat{
+                                       ID:    match[2],
+                                       Name:  name,
+                                       Group: strings.Contains(first, ", group "+match[2]+","),
+                               })
+                       } else if marked {
+                               match = whatsappImportUnknown.FindStringSubmatch(first)
+                               if len(match) > 0 {
+                                       name, day = match[1], match[2]
+                               }
+                       }
+
+                       if name == "" {
+                               return nil, errors.New("invalid WhatsApp batch header")
+                       }
+
+                       if marked {
+                               if source == 0 {
+                                       imported[name+"\n"+day] = true
+                               }
+
+                               continue
+                       }
+
+                       _, body, found := strings.Cut(entry.Text, "\n")
+                       if !found || len(body) < len("15:04 ") {
+                               return nil, errors.New("missing first WhatsApp message time")
+                       }
+
+                       date, err := time.ParseInLocation("2006-01-02 15:04", day+" "+body[:5], location)
+                       if err != nil {
+                               return nil, fmt.Errorf("parse first live message time: %w", err)
+                       }
+
+                       if cutoffs[name].IsZero() || date.Before(cutoffs[name]) {
+                               cutoffs[name] = date
+                       }
+               }
+       }
+
+       return imported, nil
+}
+

 // whatsappImportRead reads the single _chat.txt in an export without extracting
 // archive paths. Non-start lines continue the preceding message, and all
 // invisible left-to-right marks are removed. Dates use location. Media and
 // whatsappImportRead reads the single _chat.txt in an export without extracting
 // archive paths. Non-start lines continue the preceding message, and all
 // invisible left-to-right marks are removed. Dates use location. Media and