From eecbab545664bea6a95d7ef47ec927f7aae53b8f Mon Sep 17 00:00:00 2001 From: Peter Steinberger Date: Fri, 4 Sep 2026 04:31:57 -0700 Subject: [PATCH] fix(wiretap): avoid creating archives during dry runs --- CHANGELOG.md | 1 + docs/commands/wiretap.md | 3 +- internal/cli/admin_commands.go | 58 ++++++++++------- internal/cli/cli.go | 6 +- internal/cli/wiretap_dry_run_test.go | 94 ++++++++++++++++++++++++++++ 5 files changed, 134 insertions(+), 28 deletions(-) create mode 100644 internal/cli/wiretap_dry_run_test.go diff --git a/CHANGELOG.md b/CHANGELOG.md index f9eac551..8778f86e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,7 @@ - Bound release-check HTTP requests to 30 seconds with Crawlkit v0.14.8 so an unresponsive server cannot hang update checks indefinitely. - Stop guild and archived-thread pagination with a cursor error when Discord repeats a page instead of hanging sync. Thanks @SebTardif. - Reject repeated or missing message-page cursors without losing the last usable backfill checkpoint. Thanks @SebTardif. +- Keep Wiretap dry runs from creating archives or runtime directories; optional coverage reads an existing archive without migrations. ### Maintenance diff --git a/docs/commands/wiretap.md b/docs/commands/wiretap.md index 670b3c59..7773a9c2 100644 --- a/docs/commands/wiretap.md +++ b/docs/commands/wiretap.md @@ -20,7 +20,7 @@ discrawl wiretap --watch-every 10s --stats --json ## Flags - `--path ` - override the desktop data directory (default: platform-specific Discord cache path) -- `--dry-run` - report what would be imported without writing anything +- `--dry-run` - report what would be imported without creating an archive, importing data, or creating runtime directories - `--full-cache` - exhaustive Chromium HTTP cache import for historical guild-cache archaeology (slower) - `--watch-every ` - keep importing on a periodic loop - `--stats` - attach a full archive coverage snapshot; watched samples after the first include deltas @@ -37,6 +37,7 @@ discrawl wiretap --watch-every 10s --stats --json - scans local `.ldb`, `.log`, `.json`, and `.txt` artifacts for Discord message JSON, plus route-bearing Chromium HTTP cache entries by default - does not extract, store, or print Discord auth tokens - persists only compact aggregate import counters for [`coverage`](coverage.html); raw cache paths and payloads are not added to coverage state +- with `--dry-run --stats`, reads coverage from an existing archive without migrating it; reports empty coverage if no archive exists, including in watch mode. Existing archives use normal SQLite read-only access, which may create WAL/SHM sidecar files. ## Default desktop paths diff --git a/internal/cli/admin_commands.go b/internal/cli/admin_commands.go index c91396ba..e97c648f 100644 --- a/internal/cli/admin_commands.go +++ b/internal/cli/admin_commands.go @@ -403,6 +403,9 @@ func (r *runtime) runWiretap(args []string) error { if *maxFileBytes <= 0 { return usageErr(errors.New("--max-file-bytes must be positive")) } + if *watchEvery > 0 && *watchEvery < time.Second { + return usageErr(errors.New("--watch-every must be at least 1s")) + } var previousCoverage *store.CoverageReport runOnce := func(ctx context.Context) error { stats, err := discorddesktop.Import(ctx, r.store, discorddesktop.Options{ @@ -416,9 +419,12 @@ func (r *runtime) runWiretap(args []string) error { return err } if *showStats { - coverage, err := r.store.Coverage(ctx, "", r.nowUTC()) - if err != nil { - return err + coverage := store.CoverageReport{GeneratedAt: r.nowUTC(), Guilds: []store.CoverageGuild{}} + if r.store != nil { + coverage, err = r.store.Coverage(ctx, "", r.nowUTC()) + if err != nil { + return err + } } progress := wiretapProgress{Import: stats, Coverage: coverage} if previousCoverage != nil { @@ -430,29 +436,35 @@ func (r *runtime) runWiretap(args []string) error { } return r.print(stats) } - if *watchEvery <= 0 { - return runOnce(r.ctx) - } - if *watchEvery < time.Second { - return usageErr(errors.New("--watch-every must be at least 1s")) - } - ctx, stop := signal.NotifyContext(r.ctx, os.Interrupt, syscall.SIGTERM) - defer stop() - if err := runOnce(ctx); err != nil { - return err - } - ticker := time.NewTicker(*watchEvery) - defer ticker.Stop() - for { - select { - case <-ctx.Done(): - return nil - case <-ticker.C: - if err := runOnce(ctx); err != nil { - return err + run := func() error { + if *watchEvery <= 0 { + return runOnce(r.ctx) + } + ctx, stop := signal.NotifyContext(r.ctx, os.Interrupt, syscall.SIGTERM) + defer stop() + if err := runOnce(ctx); err != nil { + return err + } + ticker := time.NewTicker(*watchEvery) + defer ticker.Stop() + for { + select { + case <-ctx.Done(): + return nil + case <-ticker.C: + if err := runOnce(ctx); err != nil { + return err + } } } } + if *dryRun { + if *showStats { + return r.withLocalStoreReadOnly(run) + } + return run() + } + return r.withLocalStoreLocked(false, run) } func (r *runtime) runStatus(args []string) error { diff --git a/internal/cli/cli.go b/internal/cli/cli.go index cb356fa4..d3ddb7df 100644 --- a/internal/cli/cli.go +++ b/internal/cli/cli.go @@ -344,10 +344,8 @@ func (r *runtime) dispatch(rest []string) error { operation = "tail-failure-replay" } return r.withServicesLockedOperation(true, operation, func() error { return r.runTail(rest[1:]) }) - case "wiretap": - return r.withLocalStoreLocked(false, func() error { return r.runWiretap(rest[1:]) }) - case "tap", "cache-import": - return r.withLocalStoreLocked(false, func() error { return r.runWiretap(rest[1:]) }) + case "wiretap", "tap", "cache-import": + return r.withConfig(func() error { return r.runWiretap(rest[1:]) }) case "search": if hasHelpFlag(rest[1:]) { return printCommandUsage(r.stdout, []string{"search"}) diff --git a/internal/cli/wiretap_dry_run_test.go b/internal/cli/wiretap_dry_run_test.go new file mode 100644 index 00000000..7211edf9 --- /dev/null +++ b/internal/cli/wiretap_dry_run_test.go @@ -0,0 +1,94 @@ +package cli + +import ( + "bytes" + "context" + "crypto/sha256" + "encoding/json" + "fmt" + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/require" + + "github.com/openclaw/discrawl/internal/discorddesktop" +) + +func dryRunFixture(t *testing.T) (string, string, string) { + t.Helper() + dir := t.TempDir() + db := filepath.Join(dir, "archive", "archive.db") + desktop := filepath.Join(dir, "desktop") + require.NoError(t, os.MkdirAll(desktop, 0o755)) + require.NoError(t, os.WriteFile(filepath.Join(desktop, "payload.json"), []byte(`{"id":"111111111111111121","guild_id":"999999999999999996","type":0,"name":"synthetic"} +{"id":"333333333333333346","channel_id":"111111111111111121","content":"Sapphire preview","timestamp":"2026-09-01T12:00:00Z","author":{"id":"222222222222222232","username":"fixture"}}`), 0o600)) + cfg := filepath.Join(dir, "config.toml") + require.NoError(t, os.WriteFile(cfg, []byte(fmt.Sprintf("version = 1\ndb_path = %q\ncache_dir = %q\nlog_dir = %q\n", db, filepath.Join(dir, "runtime-cache"), filepath.Join(dir, "runtime-logs"))), 0o600)) + return cfg, db, desktop +} + +func TestWiretapDryRunDoesNotCreateArchive(t *testing.T) { + for _, command := range []string{"wiretap", "tap", "cache-import"} { + for _, stats := range []bool{false, true} { + t.Run(fmt.Sprintf("%s/stats=%t", command, stats), func(t *testing.T) { + cfg, db, desktop := dryRunFixture(t) + args := []string{"--config", cfg, "--json", command, "--dry-run", "--path", desktop} + if stats { + args = append(args, "--stats") + } + var out bytes.Buffer + require.NoError(t, Run(context.Background(), args, &out, &bytes.Buffer{})) + if stats { + var result wiretapProgress + require.NoError(t, json.Unmarshal(out.Bytes(), &result)) + require.Equal(t, 1, result.Import.Messages) + require.Zero(t, result.Coverage.Totals.MessageCount) + require.Empty(t, result.Coverage.Guilds) + } else { + var result discorddesktop.Stats + require.NoError(t, json.Unmarshal(out.Bytes(), &result)) + require.Equal(t, 1, result.Messages) + require.True(t, result.DryRun) + } + require.NoDirExists(t, filepath.Dir(db)) + require.NoDirExists(t, filepath.Join(filepath.Dir(cfg), "runtime-cache")) + require.NoDirExists(t, filepath.Join(filepath.Dir(cfg), "runtime-logs")) + }) + } + } +} + +func TestWiretapDryRunReadsExistingCoverageWithoutWriting(t *testing.T) { + cfg, db, desktop := dryRunFixture(t) + args := []string{"--config", cfg, "--json", "wiretap", "--path", desktop} + require.NoError(t, Run(context.Background(), args, &bytes.Buffer{}, &bytes.Buffer{})) + before, err := os.ReadFile(db) + require.NoError(t, err) + var out bytes.Buffer + require.NoError(t, Run(context.Background(), append(args, "--dry-run", "--stats"), &out, &bytes.Buffer{})) + var result wiretapProgress + require.NoError(t, json.Unmarshal(out.Bytes(), &result)) + require.Equal(t, 1, result.Coverage.Totals.MessageCount) + after, err := os.ReadFile(db) + require.NoError(t, err) + require.Equal(t, sha256.Sum256(before), sha256.Sum256(after)) +} + +func TestWiretapExplicitFalseDryRunStillImports(t *testing.T) { + cfg, db, desktop := dryRunFixture(t) + args := []string{"--config", cfg, "wiretap", "--path", desktop, "--dry-run", "--dry-run=false"} + require.NoError(t, Run(context.Background(), args, &bytes.Buffer{}, &bytes.Buffer{})) + require.FileExists(t, db) +} + +func TestWiretapDryRunWithoutStatsDoesNotOpenArchive(t *testing.T) { + cfg, db, desktop := dryRunFixture(t) + require.NoError(t, os.MkdirAll(filepath.Dir(db), 0o755)) + original := []byte("not a SQLite database") + require.NoError(t, os.WriteFile(db, original, 0o600)) + require.NoError(t, Run(context.Background(), []string{"--config", cfg, "wiretap", "--dry-run", "--path", desktop}, &bytes.Buffer{}, &bytes.Buffer{})) + contents, err := os.ReadFile(db) + require.NoError(t, err) + require.Equal(t, original, contents) +}