X-Git-Url: https://git.giacomolli.uk/onechat/blobdiff_plain/900ec0e7dad7a913190fcef9016e53495de9bdf8..refs/heads/master:/whatsapp_import_test.go diff --git a/whatsapp_import_test.go b/whatsapp_import_test.go index 5f7d21c..4bd0a1f 100644 --- a/whatsapp_import_test.go +++ b/whatsapp_import_test.go @@ -2,6 +2,8 @@ package main import ( "archive/zip" + "encoding/json" + "net/http" "os" "path/filepath" "strings" @@ -56,7 +58,7 @@ func TestWhatsAppImportSplitsDaysAndMapsOnlyOwner(t *testing.T) { }, } - plan, err := whatsappImportPlan(t.Context(), []string{path}, nil, names, "Sam Owner", time.UTC) + plan, err := whatsappImportPlan(t.Context(), []string{path}, nil, nil, names, "Sam Owner", time.UTC) require.NoError(t, err) require.Len(t, plan.added, 5) require.Equal(t, "2025-05-20T13:06:44Z", plan.added[0].Date) @@ -78,12 +80,12 @@ func TestWhatsAppImportSkipsHeldDaysAndMessagesAfterLiveCutoff(t *testing.T) { old := []DatedMessage{ { Kind: "work", - Text: "[WhatsApp] Live, private chat other@lid, Sunday 2025-06-08\n10:00 Ada: Later", + Text: "[WhatsApp] Book Club, private chat other@lid, Sunday 2025-06-08\n10:00 Ada: Later", Date: "2025-06-08T10:01:00Z", }, { Kind: "work", - Text: "[WhatsApp] Live, private chat other@lid, Tuesday 2025-06-03\n18:41 Ada: First\n18:42 Ada: Last", + Text: "[WhatsApp] Book Club, private chat other@lid, Tuesday 2025-06-03\n18:41 Ada: First\n18:42 Ada: Last", Date: "2025-06-03T18:43:00Z", }, { @@ -93,7 +95,7 @@ func TestWhatsAppImportSkipsHeldDaysAndMessagesAfterLiveCutoff(t *testing.T) { }, } - plan, err := whatsappImportPlan(t.Context(), []string{path}, old, map[string]WhatsAppChat{}, "Sam Owner", time.UTC) + plan, err := whatsappImportPlan(t.Context(), []string{path}, old, nil, map[string]WhatsAppChat{}, "Sam Owner", time.UTC) require.NoError(t, err) require.Equal(t, "2025-06-03T18:41:00Z", plan.cutoff.Format(time.RFC3339)) require.Equal(t, 1, plan.skipped) @@ -108,11 +110,11 @@ func TestWhatsAppImportCutoffUsesLocalTimeAndExcludesBoundary(t *testing.T) { path := whatsappImportZip(t, "Local", "[03/06/2025, 18:40:59] Ada: Before\n[03/06/2025, 18:41:00] Ada: Boundary\n[03/06/2025, 18:41:01] Ada: After\n") old := []DatedMessage{{ Kind: "work", - Text: "[WhatsApp] Live, group live@g.us, Tuesday 2025-06-03\n18:41 Ada: First", + Text: "[WhatsApp] Local, group live@g.us, Tuesday 2025-06-03\n18:41 Ada: First", Date: "2025-06-04T01:00:00Z", }} - plan, err := whatsappImportPlan(t.Context(), []string{path}, old, map[string]WhatsAppChat{}, "", time.FixedZone("TEST", -3*60*60)) + plan, err := whatsappImportPlan(t.Context(), []string{path}, old, nil, map[string]WhatsAppChat{}, "", time.FixedZone("TEST", -3*60*60)) require.NoError(t, err) require.Equal(t, "2025-06-03T18:41:00-03:00", plan.cutoff.Format(time.RFC3339)) require.Len(t, plan.added, 1) @@ -130,16 +132,16 @@ func TestWhatsAppImportKeepsSystemSenderEvenWithOwnerName(t *testing.T) { func TestWhatsAppImportRefusesOverlappingInputs(t *testing.T) { path := whatsappImportSample(t) - _, err := whatsappImportPlan(t.Context(), []string{path, path}, nil, map[string]WhatsAppChat{}, "", time.UTC) + _, err := whatsappImportPlan(t.Context(), []string{path, path}, nil, nil, map[string]WhatsAppChat{}, "", time.UTC) require.ErrorContains(t, err, "inputs overlap") } func TestWhatsAppImportSkipsAllDaysOnALaterStep(t *testing.T) { path := whatsappImportSample(t) - first, err := whatsappImportPlan(t.Context(), []string{path}, nil, nil, "Sam Owner", time.UTC) + first, err := whatsappImportPlan(t.Context(), []string{path}, nil, nil, nil, "Sam Owner", time.UTC) require.NoError(t, err) - second, err := whatsappImportPlan(t.Context(), []string{path}, first.added, nil, "Sam Owner", time.UTC) + second, err := whatsappImportPlan(t.Context(), []string{path}, first.added, nil, nil, "Sam Owner", time.UTC) require.NoError(t, err) require.Empty(t, second.added) require.Equal(t, 5, second.skipped) @@ -149,7 +151,7 @@ func TestWhatsAppImportSkipsAllDaysOnALaterStep(t *testing.T) { func TestWhatsAppImportMergesChatsByDate(t *testing.T) { later := whatsappImportZip(t, "Later", "[02/01/2025, 08:00:00] Ada: Later\n") earlier := whatsappImportZip(t, "Earlier", "[01/01/2025, 08:00:00] Bea: Earlier\n") - plan, err := whatsappImportPlan(t.Context(), []string{later, earlier}, nil, nil, "", time.UTC) + plan, err := whatsappImportPlan(t.Context(), []string{later, earlier}, nil, nil, nil, "", time.UTC) require.NoError(t, err) require.Len(t, plan.added, 2) require.Contains(t, plan.added[0].Text, "[WhatsApp] Earlier,") @@ -242,11 +244,13 @@ func TestWhatsAppImportPlanCommandDoesNotChangeAgentOrCallModel(t *testing.T) { }}) before := rebuildTestSnapshot(t, agent.dir) + var report strings.Builder + require.NoError(t, whatsappImport(t.Context(), []string{"-n", whatsappImportSample(t)}, &report)) require.Equal(t, before, rebuildTestSnapshot(t, agent.dir)) - for _, want := range []string{"chats: 1", "Book Club", "batches: 5", "dates: ", "cutoff: none", "bytes: ", "messages: 6", "estimated calls: 8", "owner: unknown"} { + for _, want := range []string{"chats: 1", "Book Club", "batches: 5", "dates: ", "cutoff: none", "bytes: ", "messages: 6", "estimated calls: 8 (about 1.3 per message, limit 1500)", "owner: unknown"} { require.Contains(t, report.String(), want) } @@ -266,15 +270,18 @@ func TestWhatsAppImportDoesNotRebuildWhenAllDaysAreHeld(t *testing.T) { agent := newRebuildTestAgent(t) dir := filepath.Dir(agent.dir) t.Setenv("ONECHAT_DIR", dir) + path := whatsappImportSample(t) - plan, err := whatsappImportPlan(t.Context(), []string{path}, nil, nil, "", time.UTC) + plan, err := whatsappImportPlan(t.Context(), []string{path}, nil, nil, nil, "", time.UTC) require.NoError(t, err) rebuildTestLog(t, agent.dir, plan.added) require.NoError(t, os.WriteFile(filepath.Join(dir, "config.json"), []byte("not model settings"), 0o600)) before := rebuildTestSnapshot(t, agent.dir) + var report strings.Builder + require.NoError(t, whatsappImport(t.Context(), []string{path}, &report)) require.Contains(t, report.String(), "batches: 0") require.Contains(t, report.String(), "estimated calls: 0") @@ -284,6 +291,7 @@ func TestWhatsAppImportDoesNotRebuildWhenAllDaysAreHeld(t *testing.T) { func TestWhatsAppImportReadsOnlyLocalSessionNames(t *testing.T) { dir := t.TempDir() whatsappTestSession(t, dir, true) + session := &WhatsApp{ dir: filepath.Join(dir, whatsappDirName), } @@ -311,6 +319,431 @@ func TestWhatsAppImportReadsOnlyLocalSessionNames(t *testing.T) { require.NotContains(t, names, "Owner LID") } +func TestWhatsAppImportCutoffUsesEachChatAndBothLogs(t *testing.T) { + path := whatsappImportZip(t, "Plans", "[03/01/2025, 09:59:59] Ada: Before\n[03/01/2025, 10:00:00] Ada: Boundary\n[03/01/2025, 10:01:00] Ada: After\n") + target := []DatedMessage{{ + Kind: "work", + Text: "[WhatsApp] Plans, group example@g.us, Friday 2025-01-03\n10:00 Ada: Target first\n10:30 Ada: Target last", + Date: "2025-01-03T10:30:00Z", + }} + + root := []DatedMessage{ + { + Kind: "work", + Text: "[WhatsApp] Other, group other@g.us, Wednesday 2025-01-01\n08:00 Bea: Other chat", + Date: "2025-01-01T08:00:00Z", + }, + { + Kind: "work", + Text: "[WhatsApp] Plans, group example@g.us, Saturday 2025-01-04\n10:00 Ada: Later batch", + Date: "2025-01-04T10:00:00Z", + }, + } + + testCases := []struct { + name string + target []DatedMessage + root []DatedMessage + }{ + { + name: "target holds the first live batch", + target: target, + root: root, + }, + { + name: "root holds the first live batch", + target: root, + root: target, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + plan, err := whatsappImportPlan(t.Context(), []string{path}, tc.target, tc.root, nil, "", time.UTC) + require.NoError(t, err) + require.Equal(t, "2025-01-03T10:00:00Z", plan.cutoff.Format(time.RFC3339)) + require.Len(t, plan.added, 1) + require.Contains(t, plan.added[0].Text, "Before") + require.NotContains(t, plan.added[0].Text, "Boundary") + require.NotContains(t, plan.added[0].Text, "After") + }) + } +} + +func TestWhatsAppImportSelectsOnlyAnExistingDirectChild(t *testing.T) { + root := newRebuildTestAgent(t) + dir := filepath.Dir(root.dir) + t.Setenv("ONECHAT_DIR", dir) + daemonChildConfig(t, dir, "House") + + before := rebuildTestSnapshot(t, root.dir) + for _, name := range []string{"Missing", "../House", root.config.Name} { + err := whatsappImport(t.Context(), []string{"-n", "--agent", name, whatsappImportSample(t)}, &strings.Builder{}) + require.ErrorContains(t, err, "does not exist") + require.Equal(t, before, rebuildTestSnapshot(t, root.dir)) + } +} + +func TestWhatsAppImportParsesGNUFlags(t *testing.T) { + testCases := []struct { + name string + args []string + want WhatsAppImportOptions + }{ + { + name: "default target remains root", + args: []string{"One.zip", "Two.zip"}, + want: WhatsAppImportOptions{paths: []string{"One.zip", "Two.zip"}}, + }, + { + name: "long flag takes a separate child name", + args: []string{"-n", "--agent", "House", "One.zip"}, + want: WhatsAppImportOptions{ + planOnly: true, + agent: "House", + paths: []string{"One.zip"}, + }, + }, + { + name: "flags can follow paths", + args: []string{"One.zip", "--agent=House", "-n"}, + want: WhatsAppImportOptions{ + planOnly: true, + agent: "House", + paths: []string{"One.zip"}, + }, + }, + { + name: "double dash ends flag parsing", + args: []string{"-n", "--", "-Example.zip"}, + want: WhatsAppImportOptions{ + planOnly: true, + paths: []string{"-Example.zip"}, + }, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + options, err := whatsappImportParse(tc.args) + require.NoError(t, err) + require.Equal(t, tc.want, options) + }) + } +} + +func TestWhatsAppImportRefusesInvalidFlags(t *testing.T) { + testCases := []struct { + name string + args []string + want string + }{ + { + name: "missing exports are refused", + args: []string{"-n", "--agent", "House"}, + want: "usage:", + }, + { + name: "missing child name is refused", + args: []string{"--agent"}, + want: "needs a child name", + }, + { + name: "empty child name is refused", + args: []string{"--agent=", "One.zip"}, + want: "needs a child name", + }, + { + name: "single dash long flag is refused", + args: []string{"-agent", "House", "One.zip"}, + want: "unknown import flag", + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + _, err := whatsappImportParse(tc.args) + require.ErrorContains(t, err, tc.want) + }) + } +} + +func TestWhatsAppImportChildPlanReadsRootCutoffButNotRootImportedDays(t *testing.T) { + root := newRebuildTestAgent(t) + dir := filepath.Dir(root.dir) + t.Setenv("ONECHAT_DIR", dir) + + childDir := daemonChildConfig(t, dir, "House") + rebuildTestLog(t, root.dir, []DatedMessage{ + { + Kind: "work", + Text: "[WhatsApp] Plans, Wednesday 2025-01-01 (imported)\n08:00 Ada: Root history", + Date: "2025-01-01T08:00:00Z", + }, + { + Kind: "work", + Text: "[WhatsApp] Plans, group example@g.us, Friday 2025-01-03\n10:00 Ada: Root live\n10:20 Ada: Last", + Date: "2025-01-03T10:20:00Z", + }, + }) + rebuildTestLog(t, childDir, []DatedMessage{ + { + Kind: "work", + Text: "[WhatsApp] Plans, Thursday 2025-01-02 (imported)\n08:00 Ada: Child history", + Date: "2025-01-02T08:00:00Z", + }, + { + Kind: "work", + Text: "[WhatsApp] Plans, group example@g.us, Saturday 2025-01-04\n10:00 Ada: Child live", + Date: "2025-01-04T10:00:00Z", + }, + }) + + path := whatsappImportZip(t, "Plans", "[01/01/2025, 08:00:00] Ada: Root history\n[02/01/2025, 08:00:00] Ada: Child history\n[03/01/2025, 09:59:59] Ada: Before\n[03/01/2025, 10:00:00] Ada: Boundary\n[03/01/2025, 10:01:00] Ada: After\n") + before := rebuildTestSnapshot(t, root.dir) + + var report strings.Builder + + require.NoError(t, whatsappImport(t.Context(), []string{"-n", "--agent", "House", path}, &report)) + require.Contains(t, report.String(), "agent: House\n") + require.Contains(t, report.String(), "batches: 2 (1 chat-days skipped)") + require.Contains(t, report.String(), "2025-01-03T10:00:00") + require.Contains(t, report.String(), "messages: 4 (merged log)") + require.Equal(t, before, rebuildTestSnapshot(t, root.dir)) + + report.Reset() + require.NoError(t, whatsappImport(t.Context(), []string{"-n", path}, &report)) + require.Contains(t, report.String(), "agent: "+root.config.Name+"\n") + require.Contains(t, report.String(), "batches: 2 (1 chat-days skipped)") + require.Equal(t, before, rebuildTestSnapshot(t, root.dir)) +} + +func TestWhatsAppImportChildRebuildUsesOwnPromptAndCache(t *testing.T) { + root := newRebuildTestAgent(t) + dir := filepath.Dir(root.dir) + childDir := daemonChildConfig(t, dir, "House") + daemonChildConfig(t, dir, "Calendar") + require.NoError(t, os.WriteFile(filepath.Join(childDir, "instructions.md"), []byte("Keep room dimensions exact."), 0o600)) + rebuildTestLog(t, root.dir, []DatedMessage{{ + Kind: "user", + Text: "Keep the root history.", + Date: "2025-02-01T00:00:00Z", + }}) + rebuildTestLog(t, childDir, []DatedMessage{{ + Kind: "user", + Text: "Keep the child history.", + Date: "2025-02-02T00:00:00Z", + }}) + + child, err := whatsappImportAgent(root.dir, "House") + require.NoError(t, err) + + old, err := rebuildReadLog(t.Context(), child.dir) + require.NoError(t, err) + + rootLog, err := rebuildReadLog(t.Context(), root.dir) + require.NoError(t, err) + + path := whatsappImportZip(t, "Plans", "[01/01/2025, 08:00:00] Ada: "+strings.Repeat("The room is ready. ", 80)+"\n") + plan, err := whatsappImportPlan(t.Context(), []string{path}, old, rootLog, nil, "", time.UTC) + require.NoError(t, err) + + before := rebuildTestSnapshot(t, child.dir) + sibling := rebuildTestSnapshot(t, filepath.Join(root.dir, "children", "Calendar")) + server := newModelFake(t) + server.reply = func(w http.ResponseWriter, _ ModelFakeCall) { + children, err := root.Children() + if err != nil || len(children) != 2 { + w.WriteHeader(http.StatusBadRequest) + + return + } + + modelWriteEvents(w, modelCompletedEvent([]any{modelMessageItem("The room is ready.")}, 100, 0, 5)) + } + + client := newModelClient(server.URL, ModelAPIKey(modelTestKey), child.CacheKey(), filepath.Join(t.TempDir(), "usage.jsonl")) + backup, err := rebuild(t.Context(), child, plan.added, client, newCompactor(client, child.Prompt, defaultConfig().Compactor, child.Tools(), child.Images()), &strings.Builder{}) + require.NoError(t, err) + require.Equal(t, before, rebuildTestSnapshot(t, backup)) + require.Equal(t, root.dir, filepath.Dir(backup)) + require.Equal(t, sibling, rebuildTestSnapshot(t, filepath.Join(root.dir, "children", "Calendar"))) + + children, err := root.Children() + require.NoError(t, err) + require.Len(t, children, 2) + + entries, err := rebuildReadLog(t.Context(), child.dir) + require.NoError(t, err) + require.Equal(t, append(plan.added, old...), entries) + + unchanged, err := rebuildReadLog(t.Context(), root.dir) + require.NoError(t, err) + require.Equal(t, rootLog, unchanged) + + prompt, err := child.Prompt() + require.NoError(t, err) + require.NotEqual(t, root.CacheKey(), child.CacheKey()) + require.Positive(t, server.Calls()) + + for i := range server.Calls() { + call := server.Call(i) + require.Equal(t, child.CacheKey(), call.body["prompt_cache_key"]) + require.Equal(t, prompt, call.body["instructions"]) + require.Contains(t, prompt, "Keep room dimensions exact.") + require.NotContains(t, string(call.raw), `"name":"whatsapp_send"`) + } +} + +func TestWhatsAppImportChildFailureKeepsConfiguredChildren(t *testing.T) { + root := newRebuildTestAgent(t) + daemonChildConfig(t, filepath.Dir(root.dir), "House") + + child, err := whatsappImportAgent(root.dir, "House") + require.NoError(t, err) + + before := rebuildTestSnapshot(t, root.dir) + server := newModelFake(t) + server.reply = func(w http.ResponseWriter, _ ModelFakeCall) { + w.WriteHeader(http.StatusBadRequest) + } + + client := newModelClient(server.URL, ModelAPIKey(modelTestKey), child.CacheKey(), filepath.Join(t.TempDir(), "usage.jsonl")) + _, err = rebuild(t.Context(), child, []DatedMessage{{ + Kind: "work", + Text: strings.Repeat("The plan is ready. ", 80), + Date: "2025-01-01T08:00:00Z", + }}, client, newCompactor(client, child.Prompt, defaultConfig().Compactor, child.Tools(), child.Images()), &strings.Builder{}) + require.ErrorContains(t, err, "rebuild") + require.Equal(t, before, rebuildTestSnapshot(t, root.dir)) + + children, err := root.Children() + require.NoError(t, err) + require.Len(t, children, 1) +} + +func TestWhatsAppImportRefusesPendingDeliveriesAndKeepsRestartSafe(t *testing.T) { + testCases := []struct { + name string + child bool + }{ + { + name: "pending child delivery keeps its log ID", + child: true, + }, + { + name: "pending root report keeps its log ID", + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + dir := t.TempDir() + t.Setenv("ONECHAT_DIR", dir) + + server := newModelFake(t) + server.reply = func(w http.ResponseWriter, _ ModelFakeCall) { + modelWriteEvents(w, modelCompletedEvent([]any{modelMessageItem("")}, 100, 0, 5)) + } + + test := newDaemonMessageTest(t, dir, server) + index := 0 + args := []string{whatsappImportZip(t, "Plans", "[01/01/2025, 08:00:00] Ada: The room is ready.\n")} + if tc.child { + index = 1 + args = append([]string{"--agent", "Garden"}, args...) + + require.NoError(t, test.messages.Tell("OneChat", "Garden", "Keep the plan.")) + } else { + require.NoError(t, test.messages.Finish(test.loops[1], nil, "The plan is ready.")) + } + + queue := test.Queue(t) + require.Len(t, queue.Pending, 1) + require.NotNil(t, queue.Pending[0].LogID) + + before, err := test.agents[index].memory.Message(*queue.Pending[0].LogID) + require.NoError(t, err) + test.Close(t) + + root := filepath.Join(dir, "agent") + snapshot := rebuildTestSnapshot(t, root) + require.NoError(t, os.WriteFile(filepath.Join(dir, "config.json"), []byte("invalid model settings"), 0o600)) + + for _, planOnly := range []bool{true, false} { + input := append([]string(nil), args...) + if planOnly { + input = append([]string{"-n"}, input...) + } + + err = whatsappImport(t.Context(), input, &strings.Builder{}) + require.ErrorContains(t, err, "has pending deliveries") + require.Equal(t, snapshot, rebuildTestSnapshot(t, root)) + require.Zero(t, server.Calls()) + } + + restarted := newDaemonMessageTest(t, dir, server) + require.Equal(t, queue, restarted.Queue(t)) + require.Equal(t, 1, restarted.agents[index].memory.Len()) + + after, err := restarted.agents[index].memory.Message(before.ID) + require.NoError(t, err) + require.Equal(t, before, after) + restarted.Start(t, index) + restarted.events[index].WaitFor(t, "idle", 1) + require.Empty(t, restarted.Queue(t).Pending) + require.Equal(t, 1, server.Calls()) + restarted.Close(t) + + again := newDaemonMessageTest(t, dir, server) + require.False(t, again.loops[index].waiting()) + require.Equal(t, 1, server.Calls()) + }) + } +} + +func TestWhatsAppImportPendingDeliveriesBlockOnlyTheirTarget(t *testing.T) { + dir := t.TempDir() + t.Setenv("ONECHAT_DIR", dir) + + server := newModelFake(t) + test := newDaemonMessageTest(t, dir, server) + require.NoError(t, test.messages.Tell("OneChat", "Garden", "Keep the plan.")) + + queue := test.Queue(t) + test.Close(t) + + path := whatsappImportZip(t, "Plans", "[01/01/2025, 08:00:00] Ada: The room is ready.\n") + for _, name := range []string{"", "Library"} { + args := []string{"-n", path} + if name != "" { + args = append(args, "--agent", name) + } + + require.NoError(t, whatsappImport(t.Context(), args, &strings.Builder{})) + } + + queue.Pending[0].LogID = nil + data, err := json.Marshal(queue) + require.NoError(t, err) + require.NoError(t, os.WriteFile(filepath.Join(dir, "agent", "messages.json"), data, 0o600)) + require.ErrorContains(t, whatsappImport(t.Context(), []string{"-n", "--agent", "Garden", path}, &strings.Builder{}), "has pending deliveries") + require.Zero(t, server.Calls()) +} + +func TestWhatsAppImportDefaultPlanDoesNotCreateAgent(t *testing.T) { + dir := t.TempDir() + t.Setenv("ONECHAT_DIR", dir) + + var report strings.Builder + + require.NoError(t, whatsappImport(t.Context(), []string{"-n", whatsappImportSample(t)}, &report)) + require.Contains(t, report.String(), "agent: OneChat\n") + + _, err := os.Stat(filepath.Join(dir, "agent")) + require.ErrorIs(t, err, os.ErrNotExist) +} + func whatsappImportSample(t *testing.T) string { t.Helper()