Use both logs for WhatsApp import cutoffs

Import plans use the first live message of each chat across the target
and root logs. An unrelated chat does not cut off an export, and only
imported days in the target count as held. This lets a child take its
history without copying newer live messages or skipping root-only days.

diff --git a/README.rst b/README.rst
index 7912d0daa3c618ac8bc0f412ed8eeeacdcd8b624..49963a10e4bae960b772da5067f0a5241dd7e83f 100644 (file)
--- a/README.rst
+++ b/README.rst
@@ -567,10 +567,11 @@ No network lookup is made.
 
 Each chat and local day becomes one work message marked ``(imported)``.
 Only messages before the first message of the earliest live WhatsApp
 
 Each chat and local day becomes one work message marked ``(imported)``.
 Only messages before the first message of the earliest live WhatsApp

-batch enter the import. Live times have minute precision, so the cutoff
-excludes that whole minute. A chat-day already imported is skipped.
-Overlapping chat-days in the supplied ZIPs are refused. Review the
-exports before import: their text goes to the model as chat history.
+batch for the same chat enter the import. Live times have minute
+precision, so the cutoff excludes that whole minute. A chat-day already
+imported is skipped. Overlapping chat-days in the supplied ZIPs are
+refused. Review the exports before import: their text goes to the model
+as chat history.

 
 The import merges the full log in date order and builds its tree with
 real compactor calls, the agent prompt, and the agent cache key. It uses
 
 The import merges the full log in date order and builds its tree with
 real compactor calls, the agent prompt, and the agent cache key. It uses

diff --git a/whatsapp_import.go b/whatsapp_import.go
index e1cbd029522810c832a35ff81e676d8fdd05a7bb..23619157ac437de4074b1b8b1f87eefca9cf3513 100644 (file)
--- a/whatsapp_import.go
+++ b/whatsapp_import.go
@@ -87,7 +87,7 @@ func whatsappImport(ctx context.Context, args []string, out io.Writer) (err erro
                return fmt.Errorf("read import names: %w", err)
        }
 
                return fmt.Errorf("read import names: %w", err)
        }
 

-       plan, err := whatsappImportPlan(ctx, args, old, names, owner, time.Local)
+       plan, err := whatsappImportPlan(ctx, args, old, nil, names, owner, time.Local)

        if err != nil {
                return fmt.Errorf("plan import: %w", err)
        }
        if err != nil {
                return fmt.Errorf("plan import: %w", err)
        }


@@ -159,6 +159,7 @@ func whatsappImport(ctx context.Context, args []string, out io.Writer) (err erro
 // log. Its estimate uses the measured cost of 1.3 calls per message, not a
 // promise about a particular export or compaction retry.
 type WhatsAppImportPlan struct {
 // log. Its estimate uses the measured cost of 1.3 calls per message, not a
 // promise about a particular export or compaction retry.
 type WhatsAppImportPlan struct {

+       cutoffs    map[string]time.Time

        added      []DatedMessage
        chats      []string
        cutoff     time.Time
        added      []DatedMessage
        chats      []string
        cutoff     time.Time


@@ -193,7 +194,13 @@ func (p WhatsAppImportPlan) Write(out io.Writer) error {
        }
 
        for _, chat := range p.chats {
        }
 
        for _, chat := range p.chats {

-               _, err = fmt.Fprintf(out, "  %s\n", chat)
+               line := "  " + chat
+               cutoff, found := p.cutoffs[chat]
+               if found {
+                       line += " (cutoff: " + cutoff.Format(time.RFC3339) + ")"
+               }
+
+               _, err = fmt.Fprintln(out, line)

                if err != nil {
                        return fmt.Errorf("write import plan: %w", err)
                }
                if err != nil {
                        return fmt.Errorf("write import plan: %w", err)
                }


@@ -319,64 +326,24 @@ func whatsappImportName(names map[string]WhatsAppChat, chat WhatsAppChat) {
        names[chat.Name] = chat
 }
 
        names[chat.Name] = chat
 }
 

-// whatsappImportPlan filters exports at the first message of the earliest live
-// batch and skips chat-days already imported. Each remaining chat-day becomes a
-// work message dated at its last exported message. Overlapping chat-days in the
-// inputs are refused, not silently combined or duplicated.
-func whatsappImportPlan(ctx context.Context, paths []string, old []DatedMessage, names map[string]WhatsAppChat, owner string, location *time.Location) (WhatsAppImportPlan, error) {
-       plan := WhatsAppImportPlan{ownerKnown: owner != ""}
+// whatsappImportPlan filters each chat at its earliest live message across the
+// target and root logs. Only imported days in the target are skipped. Each
+// remaining chat-day becomes a work message dated at its last exported message.
+// Overlapping input chat-days are refused.
+func whatsappImportPlan(ctx context.Context, paths []string, old, rootLog []DatedMessage, names map[string]WhatsAppChat, owner string, location *time.Location) (WhatsAppImportPlan, error) {
+       plan := WhatsAppImportPlan{
+               ownerKnown: owner != "",
+               cutoffs:    make(map[string]time.Time),
+       }
+

        names = maps.Clone(names)
        if names == nil {
                names = make(map[string]WhatsAppChat)
        }
 
        names = maps.Clone(names)
        if names == nil {
                names = make(map[string]WhatsAppChat)
        }
 

-       imported := make(map[string]bool)
-       for _, entry := range old {
-               if entry.Kind != "work" || !strings.HasPrefix(entry.Text, "[WhatsApp] ") {
-                       continue
-               }
-
-               first, _, _ := strings.Cut(entry.Text, "\n")
-               marked := strings.HasSuffix(first, whatsappImportMark)
-               first = strings.TrimSuffix(first, whatsappImportMark)
-               match := whatsappImportHeader.FindStringSubmatch(first)
-               name, day := "", ""
-               if len(match) > 0 {
-                       name, day = match[1], match[3]
-                       whatsappImportName(names, WhatsAppChat{
-                               ID:    match[2],
-                               Name:  name,
-                               Group: strings.Contains(first, ", group "+match[2]+","),
-                       })
-               } else if marked {
-                       match = whatsappImportUnknown.FindStringSubmatch(first)
-                       if len(match) > 0 {
-                               name, day = match[1], match[2]
-                       }
-               }
-
-               if name == "" {
-                       return plan, errors.New("read WhatsApp history: invalid batch header")
-               }
-
-               if marked {
-                       imported[name+"\n"+day] = true
-                       continue
-               }
-
-               _, body, found := strings.Cut(entry.Text, "\n")
-               if !found || len(body) < len("15:04 ") {
-                       return plan, errors.New("read WhatsApp history: missing first message time")
-               }
-
-               date, err := time.ParseInLocation("2006-01-02 15:04", day+" "+body[:5], location)
-               if err != nil {
-                       return plan, fmt.Errorf("parse first live message time: %w", err)
-               }
-
-               if plan.cutoff.IsZero() || date.Before(plan.cutoff) {
-                       plan.cutoff = date
-               }
+       imported, err := whatsappImportHistory(old, rootLog, names, plan.cutoffs, location)
+       if err != nil {
+               return plan, fmt.Errorf("read import history: %w", err)

        }
 
        seen := make(map[string]bool)
        }
 
        seen := make(map[string]bool)


@@ -399,9 +366,14 @@ func whatsappImportPlan(ctx context.Context, paths []string, old []DatedMessage,
                        plan.chats = append(plan.chats, chat.Name)
                }
 
                        plan.chats = append(plan.chats, chat.Name)
                }
 

+               cutoff := plan.cutoffs[chat.Name]
+               if !cutoff.IsZero() && (plan.cutoff.IsZero() || cutoff.Before(plan.cutoff)) {
+                       plan.cutoff = cutoff
+               }
+

                days := make(map[string][]WhatsAppMessage)
                for _, message := range messages {
                days := make(map[string][]WhatsAppMessage)
                for _, message := range messages {

-                       if !plan.cutoff.IsZero() && !message.Time.Before(plan.cutoff) {
+                       if !cutoff.IsZero() && !message.Time.Before(cutoff) {

                                continue
                        }
 
                                continue
                        }
 


@@ -451,8 +423,6 @@ func whatsappImportPlan(ctx context.Context, paths []string, old []DatedMessage,
                }
        }
 
                }
        }
 

-       var err error
-

        plan.added, err = rebuildMerge(nil, plan.added)
        if err != nil {
                return plan, fmt.Errorf("order import batches: %w", err)
        plan.added, err = rebuildMerge(nil, plan.added)
        if err != nil {
                return plan, fmt.Errorf("order import batches: %w", err)


@@ -464,6 +434,66 @@ func whatsappImportPlan(ctx context.Context, paths []string, old []DatedMessage,
        return plan, nil
 }
 
        return plan, nil
 }
 

+// whatsappImportHistory reads batch names and live cutoffs from both logs.
+// Imported chat-days belong only to the target log, which is first.
+func whatsappImportHistory(target, root []DatedMessage, names map[string]WhatsAppChat, cutoffs map[string]time.Time, location *time.Location) (map[string]bool, error) {
+       imported := make(map[string]bool)
+       for source, entries := range [][]DatedMessage{target, root} {
+               for _, entry := range entries {
+                       if entry.Kind != "work" || !strings.HasPrefix(entry.Text, "[WhatsApp] ") {
+                               continue
+                       }
+
+                       first, _, _ := strings.Cut(entry.Text, "\n")
+                       marked := strings.HasSuffix(first, whatsappImportMark)
+                       first = strings.TrimSuffix(first, whatsappImportMark)
+                       match := whatsappImportHeader.FindStringSubmatch(first)
+                       name, day := "", ""
+                       if len(match) > 0 {
+                               name, day = match[1], match[3]
+                               whatsappImportName(names, WhatsAppChat{
+                                       ID:    match[2],
+                                       Name:  name,
+                                       Group: strings.Contains(first, ", group "+match[2]+","),
+                               })
+                       } else if marked {
+                               match = whatsappImportUnknown.FindStringSubmatch(first)
+                               if len(match) > 0 {
+                                       name, day = match[1], match[2]
+                               }
+                       }
+
+                       if name == "" {
+                               return nil, errors.New("invalid WhatsApp batch header")
+                       }
+
+                       if marked {
+                               if source == 0 {
+                                       imported[name+"\n"+day] = true
+                               }
+
+                               continue
+                       }
+
+                       _, body, found := strings.Cut(entry.Text, "\n")
+                       if !found || len(body) < len("15:04 ") {
+                               return nil, errors.New("missing first WhatsApp message time")
+                       }
+
+                       date, err := time.ParseInLocation("2006-01-02 15:04", day+" "+body[:5], location)
+                       if err != nil {
+                               return nil, fmt.Errorf("parse first live message time: %w", err)
+                       }
+
+                       if cutoffs[name].IsZero() || date.Before(cutoffs[name]) {
+                               cutoffs[name] = date
+                       }
+               }
+       }
+
+       return imported, nil
+}
+

 // whatsappImportRead reads the single _chat.txt in an export without extracting
 // archive paths. Non-start lines continue the preceding message, and all
 // invisible left-to-right marks are removed. Dates use location. Media and
 // whatsappImportRead reads the single _chat.txt in an export without extracting
 // archive paths. Non-start lines continue the preceding message, and all
 // invisible left-to-right marks are removed. Dates use location. Media and

diff --git a/whatsapp_import_test.go b/whatsapp_import_test.go
index f93cd19d772112f545579268b954aa3af75c6a0f..1e24039da497f31381327604efcc6eedbe1ebada 100644 (file)
--- a/whatsapp_import_test.go
+++ b/whatsapp_import_test.go
@@ -56,7 +56,7 @@ func TestWhatsAppImportSplitsDaysAndMapsOnlyOwner(t *testing.T) {
                },
        }
 
                },
        }
 

-       plan, err := whatsappImportPlan(t.Context(), []string{path}, nil, names, "Sam Owner", time.UTC)
+       plan, err := whatsappImportPlan(t.Context(), []string{path}, nil, nil, names, "Sam Owner", time.UTC)

        require.NoError(t, err)
        require.Len(t, plan.added, 5)
        require.Equal(t, "2025-05-20T13:06:44Z", plan.added[0].Date)
        require.NoError(t, err)
        require.Len(t, plan.added, 5)
        require.Equal(t, "2025-05-20T13:06:44Z", plan.added[0].Date)


@@ -78,12 +78,12 @@ func TestWhatsAppImportSkipsHeldDaysAndMessagesAfterLiveCutoff(t *testing.T) {
        old := []DatedMessage{
                {
                        Kind: "work",
        old := []DatedMessage{
                {
                        Kind: "work",

-                       Text: "[WhatsApp] Live, private chat other@lid, Sunday 2025-06-08\n10:00 Ada: Later",
+                       Text: "[WhatsApp] Book Club, private chat other@lid, Sunday 2025-06-08\n10:00 Ada: Later",

                        Date: "2025-06-08T10:01:00Z",
                },
                {
                        Kind: "work",
                        Date: "2025-06-08T10:01:00Z",
                },
                {
                        Kind: "work",

-                       Text: "[WhatsApp] Live, private chat other@lid, Tuesday 2025-06-03\n18:41 Ada: First\n18:42 Ada: Last",
+                       Text: "[WhatsApp] Book Club, private chat other@lid, Tuesday 2025-06-03\n18:41 Ada: First\n18:42 Ada: Last",

                        Date: "2025-06-03T18:43:00Z",
                },
                {
                        Date: "2025-06-03T18:43:00Z",
                },
                {


@@ -93,7 +93,7 @@ func TestWhatsAppImportSkipsHeldDaysAndMessagesAfterLiveCutoff(t *testing.T) {
                },
        }
 
                },
        }
 

-       plan, err := whatsappImportPlan(t.Context(), []string{path}, old, map[string]WhatsAppChat{}, "Sam Owner", time.UTC)
+       plan, err := whatsappImportPlan(t.Context(), []string{path}, old, nil, map[string]WhatsAppChat{}, "Sam Owner", time.UTC)

        require.NoError(t, err)
        require.Equal(t, "2025-06-03T18:41:00Z", plan.cutoff.Format(time.RFC3339))
        require.Equal(t, 1, plan.skipped)
        require.NoError(t, err)
        require.Equal(t, "2025-06-03T18:41:00Z", plan.cutoff.Format(time.RFC3339))
        require.Equal(t, 1, plan.skipped)


@@ -108,11 +108,11 @@ func TestWhatsAppImportCutoffUsesLocalTimeAndExcludesBoundary(t *testing.T) {
        path := whatsappImportZip(t, "Local", "[03/06/2025, 18:40:59] Ada: Before\n[03/06/2025, 18:41:00] Ada: Boundary\n[03/06/2025, 18:41:01] Ada: After\n")
        old := []DatedMessage{{
                Kind: "work",
        path := whatsappImportZip(t, "Local", "[03/06/2025, 18:40:59] Ada: Before\n[03/06/2025, 18:41:00] Ada: Boundary\n[03/06/2025, 18:41:01] Ada: After\n")
        old := []DatedMessage{{
                Kind: "work",

-               Text: "[WhatsApp] Live, group live@g.us, Tuesday 2025-06-03\n18:41 Ada: First",
+               Text: "[WhatsApp] Local, group live@g.us, Tuesday 2025-06-03\n18:41 Ada: First",

                Date: "2025-06-04T01:00:00Z",
        }}
 
                Date: "2025-06-04T01:00:00Z",
        }}
 

-       plan, err := whatsappImportPlan(t.Context(), []string{path}, old, map[string]WhatsAppChat{}, "", time.FixedZone("TEST", -3*60*60))
+       plan, err := whatsappImportPlan(t.Context(), []string{path}, old, nil, map[string]WhatsAppChat{}, "", time.FixedZone("TEST", -3*60*60))

        require.NoError(t, err)
        require.Equal(t, "2025-06-03T18:41:00-03:00", plan.cutoff.Format(time.RFC3339))
        require.Len(t, plan.added, 1)
        require.NoError(t, err)
        require.Equal(t, "2025-06-03T18:41:00-03:00", plan.cutoff.Format(time.RFC3339))
        require.Len(t, plan.added, 1)


@@ -130,16 +130,16 @@ func TestWhatsAppImportKeepsSystemSenderEvenWithOwnerName(t *testing.T) {
 
 func TestWhatsAppImportRefusesOverlappingInputs(t *testing.T) {
        path := whatsappImportSample(t)
 
 func TestWhatsAppImportRefusesOverlappingInputs(t *testing.T) {
        path := whatsappImportSample(t)

-       _, err := whatsappImportPlan(t.Context(), []string{path, path}, nil, map[string]WhatsAppChat{}, "", time.UTC)
+       _, err := whatsappImportPlan(t.Context(), []string{path, path}, nil, nil, map[string]WhatsAppChat{}, "", time.UTC)

        require.ErrorContains(t, err, "inputs overlap")
 }
 
 func TestWhatsAppImportSkipsAllDaysOnALaterStep(t *testing.T) {
        path := whatsappImportSample(t)
        require.ErrorContains(t, err, "inputs overlap")
 }
 
 func TestWhatsAppImportSkipsAllDaysOnALaterStep(t *testing.T) {
        path := whatsappImportSample(t)

-       first, err := whatsappImportPlan(t.Context(), []string{path}, nil, nil, "Sam Owner", time.UTC)
+       first, err := whatsappImportPlan(t.Context(), []string{path}, nil, nil, nil, "Sam Owner", time.UTC)

        require.NoError(t, err)
 
        require.NoError(t, err)
 

-       second, err := whatsappImportPlan(t.Context(), []string{path}, first.added, nil, "Sam Owner", time.UTC)
+       second, err := whatsappImportPlan(t.Context(), []string{path}, first.added, nil, nil, "Sam Owner", time.UTC)

        require.NoError(t, err)
        require.Empty(t, second.added)
        require.Equal(t, 5, second.skipped)
        require.NoError(t, err)
        require.Empty(t, second.added)
        require.Equal(t, 5, second.skipped)


@@ -149,7 +149,7 @@ func TestWhatsAppImportSkipsAllDaysOnALaterStep(t *testing.T) {
 func TestWhatsAppImportMergesChatsByDate(t *testing.T) {
        later := whatsappImportZip(t, "Later", "[02/01/2025, 08:00:00] Ada: Later\n")
        earlier := whatsappImportZip(t, "Earlier", "[01/01/2025, 08:00:00] Bea: Earlier\n")
 func TestWhatsAppImportMergesChatsByDate(t *testing.T) {
        later := whatsappImportZip(t, "Later", "[02/01/2025, 08:00:00] Ada: Later\n")
        earlier := whatsappImportZip(t, "Earlier", "[01/01/2025, 08:00:00] Bea: Earlier\n")

-       plan, err := whatsappImportPlan(t.Context(), []string{later, earlier}, nil, nil, "", time.UTC)
+       plan, err := whatsappImportPlan(t.Context(), []string{later, earlier}, nil, nil, nil, "", time.UTC)

        require.NoError(t, err)
        require.Len(t, plan.added, 2)
        require.Contains(t, plan.added[0].Text, "[WhatsApp] Earlier,")
        require.NoError(t, err)
        require.Len(t, plan.added, 2)
        require.Contains(t, plan.added[0].Text, "[WhatsApp] Earlier,")


@@ -270,7 +270,7 @@ func TestWhatsAppImportDoesNotRebuildWhenAllDaysAreHeld(t *testing.T) {
        t.Setenv("ONECHAT_DIR", dir)
 
        path := whatsappImportSample(t)
        t.Setenv("ONECHAT_DIR", dir)
 
        path := whatsappImportSample(t)

-       plan, err := whatsappImportPlan(t.Context(), []string{path}, nil, nil, "", time.UTC)
+       plan, err := whatsappImportPlan(t.Context(), []string{path}, nil, nil, nil, "", time.UTC)

        require.NoError(t, err)
 
        rebuildTestLog(t, agent.dir, plan.added)
        require.NoError(t, err)
 
        rebuildTestLog(t, agent.dir, plan.added)


@@ -317,6 +317,57 @@ func TestWhatsAppImportReadsOnlyLocalSessionNames(t *testing.T) {
        require.NotContains(t, names, "Owner LID")
 }
 
        require.NotContains(t, names, "Owner LID")
 }
 

+func TestWhatsAppImportCutoffUsesEachChatAndBothLogs(t *testing.T) {
+       path := whatsappImportZip(t, "Plans", "[03/01/2025, 09:59:59] Ada: Before\n[03/01/2025, 10:00:00] Ada: Boundary\n[03/01/2025, 10:01:00] Ada: After\n")
+       target := []DatedMessage{{
+               Kind: "work",
+               Text: "[WhatsApp] Plans, group example@g.us, Friday 2025-01-03\n10:00 Ada: Target first\n10:30 Ada: Target last",
+               Date: "2025-01-03T10:30:00Z",
+       }}
+
+       root := []DatedMessage{
+               {
+                       Kind: "work",
+                       Text: "[WhatsApp] Other, group other@g.us, Wednesday 2025-01-01\n08:00 Bea: Other chat",
+                       Date: "2025-01-01T08:00:00Z",
+               },
+               {
+                       Kind: "work",
+                       Text: "[WhatsApp] Plans, group example@g.us, Saturday 2025-01-04\n10:00 Ada: Later batch",
+                       Date: "2025-01-04T10:00:00Z",
+               },
+       }
+
+       testCases := []struct {
+               name   string
+               target []DatedMessage
+               root   []DatedMessage
+       }{
+               {
+                       name:   "target holds the first live batch",
+                       target: target,
+                       root:   root,
+               },
+               {
+                       name:   "root holds the first live batch",
+                       target: root,
+                       root:   target,
+               },
+       }
+
+       for _, tc := range testCases {
+               t.Run(tc.name, func(t *testing.T) {
+                       plan, err := whatsappImportPlan(t.Context(), []string{path}, tc.target, tc.root, nil, "", time.UTC)
+                       require.NoError(t, err)
+                       require.Equal(t, "2025-01-03T10:00:00Z", plan.cutoff.Format(time.RFC3339))
+                       require.Len(t, plan.added, 1)
+                       require.Contains(t, plan.added[0].Text, "Before")
+                       require.NotContains(t, plan.added[0].Text, "Boundary")
+                       require.NotContains(t, plan.added[0].Text, "After")
+               })
+       }
+}
+

 func whatsappImportSample(t *testing.T) string {
        t.Helper()
 
 func whatsappImportSample(t *testing.T) string {
        t.Helper()