X-Git-Url: https://git.giacomolli.uk/onechat/blobdiff_plain/900ec0e7dad7a913190fcef9016e53495de9bdf8..855a45e6448537c3eed1ad6ec7f1ca9d6caf91fc:/whatsapp_import.go?ds=sidebyside

diff --git a/whatsapp_import.go b/whatsapp_import.go
index 062d3ca..2361915 100644
--- a/whatsapp_import.go
+++ b/whatsapp_import.go
@@ -87,7 +87,7 @@ func whatsappImport(ctx context.Context, args []string, out io.Writer) (err erro
 		return fmt.Errorf("read import names: %w", err)
 	}
 
-	plan, err := whatsappImportPlan(ctx, args, old, names, owner, time.Local)
+	plan, err := whatsappImportPlan(ctx, args, old, nil, names, owner, time.Local)
 	if err != nil {
 		return fmt.Errorf("plan import: %w", err)
 	}
@@ -142,7 +142,7 @@ func whatsappImport(ctx context.Context, args []string, out io.Writer) (err erro
 	}
 
 	client := newModelClient(address, auth, agent.CacheKey(), usagePath)
-	backup, err := rebuild(ctx, agent, plan.added, client, newCompactor(client, agent.Prompt, config.Compactor), out)
+	backup, err := rebuild(ctx, agent, plan.added, client, newCompactor(client, agent.Prompt, config.Compactor, agent.Tools()), out)
 	if err != nil {
 		return fmt.Errorf("rebuild imported memory: %w", err)
 	}
@@ -159,6 +159,7 @@ func whatsappImport(ctx context.Context, args []string, out io.Writer) (err erro
 // log. Its estimate uses the measured cost of 1.3 calls per message, not a
 // promise about a particular export or compaction retry.
 type WhatsAppImportPlan struct {
+	cutoffs    map[string]time.Time
 	added      []DatedMessage
 	chats      []string
 	cutoff     time.Time
@@ -193,7 +194,13 @@ func (p WhatsAppImportPlan) Write(out io.Writer) error {
 	}
 
 	for _, chat := range p.chats {
-		_, err = fmt.Fprintf(out, "  %s\n", chat)
+		line := "  " + chat
+		cutoff, found := p.cutoffs[chat]
+		if found {
+			line += " (cutoff: " + cutoff.Format(time.RFC3339) + ")"
+		}
+
+		_, err = fmt.Fprintln(out, line)
 		if err != nil {
 			return fmt.Errorf("write import plan: %w", err)
 		}
@@ -211,7 +218,7 @@ func (p WhatsAppImportPlan) Write(out io.Writer) error {
 		"owner: " + owner,
 		fmt.Sprintf("bytes: %d (new batch text)", p.bytes),
 		fmt.Sprintf("messages: %d (merged log)", p.messages),
-		fmt.Sprintf("estimated calls: %.0f (about 1.3 per message, limit %d)", estimate, buildCallLimit),
+		fmt.Sprintf("estimated calls: %.0f (about 1.3 per message, limit %d)", estimate, rebuildCallLimit),
 	}
 
 	_, err = io.WriteString(out, strings.Join(lines, "\n")+"\n")
@@ -296,6 +303,7 @@ func whatsappImportNames(ctx context.Context, dir string) (names map[string]What
 	}
 
 	var state WhatsAppStateFile
+
 	err = json.Unmarshal(data, &state)
 	if err != nil {
 		return nil, "", fmt.Errorf("decode pending import names: %w", err)
@@ -318,64 +326,24 @@ func whatsappImportName(names map[string]WhatsAppChat, chat WhatsAppChat) {
 	names[chat.Name] = chat
 }
 
-// whatsappImportPlan filters exports at the first message of the earliest live
-// batch and skips chat-days already imported. Each remaining chat-day becomes a
-// work message dated at its last exported message. Overlapping chat-days in the
-// inputs are refused, not silently combined or duplicated.
-func whatsappImportPlan(ctx context.Context, paths []string, old []DatedMessage, names map[string]WhatsAppChat, owner string, location *time.Location) (WhatsAppImportPlan, error) {
-	plan := WhatsAppImportPlan{ownerKnown: owner != ""}
+// whatsappImportPlan filters each chat at its earliest live message across the
+// target and root logs. Only imported days in the target are skipped. Each
+// remaining chat-day becomes a work message dated at its last exported message.
+// Overlapping input chat-days are refused.
+func whatsappImportPlan(ctx context.Context, paths []string, old, rootLog []DatedMessage, names map[string]WhatsAppChat, owner string, location *time.Location) (WhatsAppImportPlan, error) {
+	plan := WhatsAppImportPlan{
+		ownerKnown: owner != "",
+		cutoffs:    make(map[string]time.Time),
+	}
+
 	names = maps.Clone(names)
 	if names == nil {
 		names = make(map[string]WhatsAppChat)
 	}
 
-	imported := make(map[string]bool)
-	for _, entry := range old {
-		if entry.Kind != "work" || !strings.HasPrefix(entry.Text, "[WhatsApp] ") {
-			continue
-		}
-
-		first, _, _ := strings.Cut(entry.Text, "\n")
-		marked := strings.HasSuffix(first, whatsappImportMark)
-		first = strings.TrimSuffix(first, whatsappImportMark)
-		match := whatsappImportHeader.FindStringSubmatch(first)
-		name, day := "", ""
-		if len(match) > 0 {
-			name, day = match[1], match[3]
-			whatsappImportName(names, WhatsAppChat{
-				ID:    match[2],
-				Name:  name,
-				Group: strings.Contains(first, ", group "+match[2]+","),
-			})
-		} else if marked {
-			match = whatsappImportUnknown.FindStringSubmatch(first)
-			if len(match) > 0 {
-				name, day = match[1], match[2]
-			}
-		}
-
-		if name == "" {
-			return plan, errors.New("read WhatsApp history: invalid batch header")
-		}
-
-		if marked {
-			imported[name+"\n"+day] = true
-			continue
-		}
-
-		_, body, found := strings.Cut(entry.Text, "\n")
-		if !found || len(body) < len("15:04 ") {
-			return plan, errors.New("read WhatsApp history: missing first message time")
-		}
-
-		date, err := time.ParseInLocation("2006-01-02 15:04", day+" "+body[:5], location)
-		if err != nil {
-			return plan, fmt.Errorf("parse first live message time: %w", err)
-		}
-
-		if plan.cutoff.IsZero() || date.Before(plan.cutoff) {
-			plan.cutoff = date
-		}
+	imported, err := whatsappImportHistory(old, rootLog, names, plan.cutoffs, location)
+	if err != nil {
+		return plan, fmt.Errorf("read import history: %w", err)
 	}
 
 	seen := make(map[string]bool)
@@ -398,9 +366,14 @@ func whatsappImportPlan(ctx context.Context, paths []string, old []DatedMessage,
 			plan.chats = append(plan.chats, chat.Name)
 		}
 
+		cutoff := plan.cutoffs[chat.Name]
+		if !cutoff.IsZero() && (plan.cutoff.IsZero() || cutoff.Before(plan.cutoff)) {
+			plan.cutoff = cutoff
+		}
+
 		days := make(map[string][]WhatsAppMessage)
 		for _, message := range messages {
-			if !plan.cutoff.IsZero() && !message.Time.Before(plan.cutoff) {
+			if !cutoff.IsZero() && !message.Time.Before(cutoff) {
 				continue
 			}
 
@@ -450,7 +423,6 @@ func whatsappImportPlan(ctx context.Context, paths []string, old []DatedMessage,
 		}
 	}
 
-	var err error
 	plan.added, err = rebuildMerge(nil, plan.added)
 	if err != nil {
 		return plan, fmt.Errorf("order import batches: %w", err)
@@ -462,6 +434,66 @@ func whatsappImportPlan(ctx context.Context, paths []string, old []DatedMessage,
 	return plan, nil
 }
 
+// whatsappImportHistory reads batch names and live cutoffs from both logs.
+// Imported chat-days belong only to the target log, which is first.
+func whatsappImportHistory(target, root []DatedMessage, names map[string]WhatsAppChat, cutoffs map[string]time.Time, location *time.Location) (map[string]bool, error) {
+	imported := make(map[string]bool)
+	for source, entries := range [][]DatedMessage{target, root} {
+		for _, entry := range entries {
+			if entry.Kind != "work" || !strings.HasPrefix(entry.Text, "[WhatsApp] ") {
+				continue
+			}
+
+			first, _, _ := strings.Cut(entry.Text, "\n")
+			marked := strings.HasSuffix(first, whatsappImportMark)
+			first = strings.TrimSuffix(first, whatsappImportMark)
+			match := whatsappImportHeader.FindStringSubmatch(first)
+			name, day := "", ""
+			if len(match) > 0 {
+				name, day = match[1], match[3]
+				whatsappImportName(names, WhatsAppChat{
+					ID:    match[2],
+					Name:  name,
+					Group: strings.Contains(first, ", group "+match[2]+","),
+				})
+			} else if marked {
+				match = whatsappImportUnknown.FindStringSubmatch(first)
+				if len(match) > 0 {
+					name, day = match[1], match[2]
+				}
+			}
+
+			if name == "" {
+				return nil, errors.New("invalid WhatsApp batch header")
+			}
+
+			if marked {
+				if source == 0 {
+					imported[name+"\n"+day] = true
+				}
+
+				continue
+			}
+
+			_, body, found := strings.Cut(entry.Text, "\n")
+			if !found || len(body) < len("15:04 ") {
+				return nil, errors.New("missing first WhatsApp message time")
+			}
+
+			date, err := time.ParseInLocation("2006-01-02 15:04", day+" "+body[:5], location)
+			if err != nil {
+				return nil, fmt.Errorf("parse first live message time: %w", err)
+			}
+
+			if cutoffs[name].IsZero() || date.Before(cutoffs[name]) {
+				cutoffs[name] = date
+			}
+		}
+	}
+
+	return imported, nil
+}
+
 // whatsappImportRead reads the single _chat.txt in an export without extracting
 // archive paths. Non-start lines continue the preceding message, and all
 // invisible left-to-right marks are removed. Dates use location. Media and
@@ -485,6 +517,7 @@ func whatsappImportRead(path, owner string, location *time.Location) (chat Whats
 	}()
 
 	var file *zip.File
+
 	for _, entry := range archive.File {
 		if filepath.Base(entry.Name) == "_chat.txt" && !entry.FileInfo().IsDir() {
 			if file != nil {
@@ -517,6 +550,7 @@ func whatsappImportRead(path, owner string, location *time.Location) (chat Whats
 
 	scanner := bufio.NewScanner(io.LimitReader(reader, whatsappImportBytes+1))
 	scanner.Buffer(make([]byte, 64*1024), buildLineBytes)
+
 	for scanner.Scan() {
 		line := strings.ReplaceAll(scanner.Text(), "\u200e", "")
 		match := whatsappImportStart.FindStringSubmatch(line)
@@ -530,6 +564,7 @@ func whatsappImportRead(path, owner string, location *time.Location) (chat Whats
 			}
 
 			messages[len(messages)-1].Text += "\n" + line
+
 			continue
 		}