From: Anderson A. Giacomolli Date: Tue, 10 Nov 2026 10:43:34 +0000 (-0300) Subject: Use both logs for WhatsApp import cutoffs X-Git-Url: http://git.giacomolli.uk/onechat/commitdiff_plain/855a45e6448537c3eed1ad6ec7f1ca9d6caf91fc Use both logs for WhatsApp import cutoffs Import plans use the first live message of each chat across the target and root logs. An unrelated chat does not cut off an export, and only imported days in the target count as held. This lets a child take its history without copying newer live messages or skipping root-only days. --- diff --git a/README.rst b/README.rst index 7912d0d..49963a1 100644 --- a/README.rst +++ b/README.rst @@ -567,10 +567,11 @@ No network lookup is made. Each chat and local day becomes one work message marked ``(imported)``. Only messages before the first message of the earliest live WhatsApp -batch enter the import. Live times have minute precision, so the cutoff -excludes that whole minute. A chat-day already imported is skipped. -Overlapping chat-days in the supplied ZIPs are refused. Review the -exports before import: their text goes to the model as chat history. +batch for the same chat enter the import. Live times have minute +precision, so the cutoff excludes that whole minute. A chat-day already +imported is skipped. Overlapping chat-days in the supplied ZIPs are +refused. Review the exports before import: their text goes to the model +as chat history. The import merges the full log in date order and builds its tree with real compactor calls, the agent prompt, and the agent cache key. It uses diff --git a/whatsapp_import.go b/whatsapp_import.go index e1cbd02..2361915 100644 --- a/whatsapp_import.go +++ b/whatsapp_import.go @@ -87,7 +87,7 @@ func whatsappImport(ctx context.Context, args []string, out io.Writer) (err erro return fmt.Errorf("read import names: %w", err) } - plan, err := whatsappImportPlan(ctx, args, old, names, owner, time.Local) + plan, err := whatsappImportPlan(ctx, args, old, nil, names, owner, time.Local) if err != nil { return fmt.Errorf("plan import: %w", err) } @@ -159,6 +159,7 @@ func whatsappImport(ctx context.Context, args []string, out io.Writer) (err erro // log. Its estimate uses the measured cost of 1.3 calls per message, not a // promise about a particular export or compaction retry. type WhatsAppImportPlan struct { + cutoffs map[string]time.Time added []DatedMessage chats []string cutoff time.Time @@ -193,7 +194,13 @@ func (p WhatsAppImportPlan) Write(out io.Writer) error { } for _, chat := range p.chats { - _, err = fmt.Fprintf(out, " %s\n", chat) + line := " " + chat + cutoff, found := p.cutoffs[chat] + if found { + line += " (cutoff: " + cutoff.Format(time.RFC3339) + ")" + } + + _, err = fmt.Fprintln(out, line) if err != nil { return fmt.Errorf("write import plan: %w", err) } @@ -319,64 +326,24 @@ func whatsappImportName(names map[string]WhatsAppChat, chat WhatsAppChat) { names[chat.Name] = chat } -// whatsappImportPlan filters exports at the first message of the earliest live -// batch and skips chat-days already imported. Each remaining chat-day becomes a -// work message dated at its last exported message. Overlapping chat-days in the -// inputs are refused, not silently combined or duplicated. -func whatsappImportPlan(ctx context.Context, paths []string, old []DatedMessage, names map[string]WhatsAppChat, owner string, location *time.Location) (WhatsAppImportPlan, error) { - plan := WhatsAppImportPlan{ownerKnown: owner != ""} +// whatsappImportPlan filters each chat at its earliest live message across the +// target and root logs. Only imported days in the target are skipped. Each +// remaining chat-day becomes a work message dated at its last exported message. +// Overlapping input chat-days are refused. +func whatsappImportPlan(ctx context.Context, paths []string, old, rootLog []DatedMessage, names map[string]WhatsAppChat, owner string, location *time.Location) (WhatsAppImportPlan, error) { + plan := WhatsAppImportPlan{ + ownerKnown: owner != "", + cutoffs: make(map[string]time.Time), + } + names = maps.Clone(names) if names == nil { names = make(map[string]WhatsAppChat) } - imported := make(map[string]bool) - for _, entry := range old { - if entry.Kind != "work" || !strings.HasPrefix(entry.Text, "[WhatsApp] ") { - continue - } - - first, _, _ := strings.Cut(entry.Text, "\n") - marked := strings.HasSuffix(first, whatsappImportMark) - first = strings.TrimSuffix(first, whatsappImportMark) - match := whatsappImportHeader.FindStringSubmatch(first) - name, day := "", "" - if len(match) > 0 { - name, day = match[1], match[3] - whatsappImportName(names, WhatsAppChat{ - ID: match[2], - Name: name, - Group: strings.Contains(first, ", group "+match[2]+","), - }) - } else if marked { - match = whatsappImportUnknown.FindStringSubmatch(first) - if len(match) > 0 { - name, day = match[1], match[2] - } - } - - if name == "" { - return plan, errors.New("read WhatsApp history: invalid batch header") - } - - if marked { - imported[name+"\n"+day] = true - continue - } - - _, body, found := strings.Cut(entry.Text, "\n") - if !found || len(body) < len("15:04 ") { - return plan, errors.New("read WhatsApp history: missing first message time") - } - - date, err := time.ParseInLocation("2006-01-02 15:04", day+" "+body[:5], location) - if err != nil { - return plan, fmt.Errorf("parse first live message time: %w", err) - } - - if plan.cutoff.IsZero() || date.Before(plan.cutoff) { - plan.cutoff = date - } + imported, err := whatsappImportHistory(old, rootLog, names, plan.cutoffs, location) + if err != nil { + return plan, fmt.Errorf("read import history: %w", err) } seen := make(map[string]bool) @@ -399,9 +366,14 @@ func whatsappImportPlan(ctx context.Context, paths []string, old []DatedMessage, plan.chats = append(plan.chats, chat.Name) } + cutoff := plan.cutoffs[chat.Name] + if !cutoff.IsZero() && (plan.cutoff.IsZero() || cutoff.Before(plan.cutoff)) { + plan.cutoff = cutoff + } + days := make(map[string][]WhatsAppMessage) for _, message := range messages { - if !plan.cutoff.IsZero() && !message.Time.Before(plan.cutoff) { + if !cutoff.IsZero() && !message.Time.Before(cutoff) { continue } @@ -451,8 +423,6 @@ func whatsappImportPlan(ctx context.Context, paths []string, old []DatedMessage, } } - var err error - plan.added, err = rebuildMerge(nil, plan.added) if err != nil { return plan, fmt.Errorf("order import batches: %w", err) @@ -464,6 +434,66 @@ func whatsappImportPlan(ctx context.Context, paths []string, old []DatedMessage, return plan, nil } +// whatsappImportHistory reads batch names and live cutoffs from both logs. +// Imported chat-days belong only to the target log, which is first. +func whatsappImportHistory(target, root []DatedMessage, names map[string]WhatsAppChat, cutoffs map[string]time.Time, location *time.Location) (map[string]bool, error) { + imported := make(map[string]bool) + for source, entries := range [][]DatedMessage{target, root} { + for _, entry := range entries { + if entry.Kind != "work" || !strings.HasPrefix(entry.Text, "[WhatsApp] ") { + continue + } + + first, _, _ := strings.Cut(entry.Text, "\n") + marked := strings.HasSuffix(first, whatsappImportMark) + first = strings.TrimSuffix(first, whatsappImportMark) + match := whatsappImportHeader.FindStringSubmatch(first) + name, day := "", "" + if len(match) > 0 { + name, day = match[1], match[3] + whatsappImportName(names, WhatsAppChat{ + ID: match[2], + Name: name, + Group: strings.Contains(first, ", group "+match[2]+","), + }) + } else if marked { + match = whatsappImportUnknown.FindStringSubmatch(first) + if len(match) > 0 { + name, day = match[1], match[2] + } + } + + if name == "" { + return nil, errors.New("invalid WhatsApp batch header") + } + + if marked { + if source == 0 { + imported[name+"\n"+day] = true + } + + continue + } + + _, body, found := strings.Cut(entry.Text, "\n") + if !found || len(body) < len("15:04 ") { + return nil, errors.New("missing first WhatsApp message time") + } + + date, err := time.ParseInLocation("2006-01-02 15:04", day+" "+body[:5], location) + if err != nil { + return nil, fmt.Errorf("parse first live message time: %w", err) + } + + if cutoffs[name].IsZero() || date.Before(cutoffs[name]) { + cutoffs[name] = date + } + } + } + + return imported, nil +} + // whatsappImportRead reads the single _chat.txt in an export without extracting // archive paths. Non-start lines continue the preceding message, and all // invisible left-to-right marks are removed. Dates use location. Media and diff --git a/whatsapp_import_test.go b/whatsapp_import_test.go index f93cd19..1e24039 100644 --- a/whatsapp_import_test.go +++ b/whatsapp_import_test.go @@ -56,7 +56,7 @@ func TestWhatsAppImportSplitsDaysAndMapsOnlyOwner(t *testing.T) { }, } - plan, err := whatsappImportPlan(t.Context(), []string{path}, nil, names, "Sam Owner", time.UTC) + plan, err := whatsappImportPlan(t.Context(), []string{path}, nil, nil, names, "Sam Owner", time.UTC) require.NoError(t, err) require.Len(t, plan.added, 5) require.Equal(t, "2025-05-20T13:06:44Z", plan.added[0].Date) @@ -78,12 +78,12 @@ func TestWhatsAppImportSkipsHeldDaysAndMessagesAfterLiveCutoff(t *testing.T) { old := []DatedMessage{ { Kind: "work", - Text: "[WhatsApp] Live, private chat other@lid, Sunday 2025-06-08\n10:00 Ada: Later", + Text: "[WhatsApp] Book Club, private chat other@lid, Sunday 2025-06-08\n10:00 Ada: Later", Date: "2025-06-08T10:01:00Z", }, { Kind: "work", - Text: "[WhatsApp] Live, private chat other@lid, Tuesday 2025-06-03\n18:41 Ada: First\n18:42 Ada: Last", + Text: "[WhatsApp] Book Club, private chat other@lid, Tuesday 2025-06-03\n18:41 Ada: First\n18:42 Ada: Last", Date: "2025-06-03T18:43:00Z", }, { @@ -93,7 +93,7 @@ func TestWhatsAppImportSkipsHeldDaysAndMessagesAfterLiveCutoff(t *testing.T) { }, } - plan, err := whatsappImportPlan(t.Context(), []string{path}, old, map[string]WhatsAppChat{}, "Sam Owner", time.UTC) + plan, err := whatsappImportPlan(t.Context(), []string{path}, old, nil, map[string]WhatsAppChat{}, "Sam Owner", time.UTC) require.NoError(t, err) require.Equal(t, "2025-06-03T18:41:00Z", plan.cutoff.Format(time.RFC3339)) require.Equal(t, 1, plan.skipped) @@ -108,11 +108,11 @@ func TestWhatsAppImportCutoffUsesLocalTimeAndExcludesBoundary(t *testing.T) { path := whatsappImportZip(t, "Local", "[03/06/2025, 18:40:59] Ada: Before\n[03/06/2025, 18:41:00] Ada: Boundary\n[03/06/2025, 18:41:01] Ada: After\n") old := []DatedMessage{{ Kind: "work", - Text: "[WhatsApp] Live, group live@g.us, Tuesday 2025-06-03\n18:41 Ada: First", + Text: "[WhatsApp] Local, group live@g.us, Tuesday 2025-06-03\n18:41 Ada: First", Date: "2025-06-04T01:00:00Z", }} - plan, err := whatsappImportPlan(t.Context(), []string{path}, old, map[string]WhatsAppChat{}, "", time.FixedZone("TEST", -3*60*60)) + plan, err := whatsappImportPlan(t.Context(), []string{path}, old, nil, map[string]WhatsAppChat{}, "", time.FixedZone("TEST", -3*60*60)) require.NoError(t, err) require.Equal(t, "2025-06-03T18:41:00-03:00", plan.cutoff.Format(time.RFC3339)) require.Len(t, plan.added, 1) @@ -130,16 +130,16 @@ func TestWhatsAppImportKeepsSystemSenderEvenWithOwnerName(t *testing.T) { func TestWhatsAppImportRefusesOverlappingInputs(t *testing.T) { path := whatsappImportSample(t) - _, err := whatsappImportPlan(t.Context(), []string{path, path}, nil, map[string]WhatsAppChat{}, "", time.UTC) + _, err := whatsappImportPlan(t.Context(), []string{path, path}, nil, nil, map[string]WhatsAppChat{}, "", time.UTC) require.ErrorContains(t, err, "inputs overlap") } func TestWhatsAppImportSkipsAllDaysOnALaterStep(t *testing.T) { path := whatsappImportSample(t) - first, err := whatsappImportPlan(t.Context(), []string{path}, nil, nil, "Sam Owner", time.UTC) + first, err := whatsappImportPlan(t.Context(), []string{path}, nil, nil, nil, "Sam Owner", time.UTC) require.NoError(t, err) - second, err := whatsappImportPlan(t.Context(), []string{path}, first.added, nil, "Sam Owner", time.UTC) + second, err := whatsappImportPlan(t.Context(), []string{path}, first.added, nil, nil, "Sam Owner", time.UTC) require.NoError(t, err) require.Empty(t, second.added) require.Equal(t, 5, second.skipped) @@ -149,7 +149,7 @@ func TestWhatsAppImportSkipsAllDaysOnALaterStep(t *testing.T) { func TestWhatsAppImportMergesChatsByDate(t *testing.T) { later := whatsappImportZip(t, "Later", "[02/01/2025, 08:00:00] Ada: Later\n") earlier := whatsappImportZip(t, "Earlier", "[01/01/2025, 08:00:00] Bea: Earlier\n") - plan, err := whatsappImportPlan(t.Context(), []string{later, earlier}, nil, nil, "", time.UTC) + plan, err := whatsappImportPlan(t.Context(), []string{later, earlier}, nil, nil, nil, "", time.UTC) require.NoError(t, err) require.Len(t, plan.added, 2) require.Contains(t, plan.added[0].Text, "[WhatsApp] Earlier,") @@ -270,7 +270,7 @@ func TestWhatsAppImportDoesNotRebuildWhenAllDaysAreHeld(t *testing.T) { t.Setenv("ONECHAT_DIR", dir) path := whatsappImportSample(t) - plan, err := whatsappImportPlan(t.Context(), []string{path}, nil, nil, "", time.UTC) + plan, err := whatsappImportPlan(t.Context(), []string{path}, nil, nil, nil, "", time.UTC) require.NoError(t, err) rebuildTestLog(t, agent.dir, plan.added) @@ -317,6 +317,57 @@ func TestWhatsAppImportReadsOnlyLocalSessionNames(t *testing.T) { require.NotContains(t, names, "Owner LID") } +func TestWhatsAppImportCutoffUsesEachChatAndBothLogs(t *testing.T) { + path := whatsappImportZip(t, "Plans", "[03/01/2025, 09:59:59] Ada: Before\n[03/01/2025, 10:00:00] Ada: Boundary\n[03/01/2025, 10:01:00] Ada: After\n") + target := []DatedMessage{{ + Kind: "work", + Text: "[WhatsApp] Plans, group example@g.us, Friday 2025-01-03\n10:00 Ada: Target first\n10:30 Ada: Target last", + Date: "2025-01-03T10:30:00Z", + }} + + root := []DatedMessage{ + { + Kind: "work", + Text: "[WhatsApp] Other, group other@g.us, Wednesday 2025-01-01\n08:00 Bea: Other chat", + Date: "2025-01-01T08:00:00Z", + }, + { + Kind: "work", + Text: "[WhatsApp] Plans, group example@g.us, Saturday 2025-01-04\n10:00 Ada: Later batch", + Date: "2025-01-04T10:00:00Z", + }, + } + + testCases := []struct { + name string + target []DatedMessage + root []DatedMessage + }{ + { + name: "target holds the first live batch", + target: target, + root: root, + }, + { + name: "root holds the first live batch", + target: root, + root: target, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + plan, err := whatsappImportPlan(t.Context(), []string{path}, tc.target, tc.root, nil, "", time.UTC) + require.NoError(t, err) + require.Equal(t, "2025-01-03T10:00:00Z", plan.cutoff.Format(time.RFC3339)) + require.Len(t, plan.added, 1) + require.Contains(t, plan.added[0].Text, "Before") + require.NotContains(t, plan.added[0].Text, "Boundary") + require.NotContains(t, plan.added[0].Text, "After") + }) + } +} + func whatsappImportSample(t *testing.T) string { t.Helper()