Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@
- Bound release-check HTTP requests to 30 seconds with Crawlkit v0.14.8 so an unresponsive server cannot hang update checks indefinitely.
- Stop guild and archived-thread pagination with a cursor error when Discord repeats a page instead of hanging sync. Thanks @SebTardif.
- Reject repeated or missing message-page cursors without losing the last usable backfill checkpoint. Thanks @SebTardif.
- Keep Wiretap dry runs from creating archives or runtime directories; optional coverage reads an existing archive without migrations.

### Maintenance

Expand Down
3 changes: 2 additions & 1 deletion docs/commands/wiretap.md
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,7 @@ discrawl wiretap --watch-every 10s --stats --json
## Flags

- `--path <dir>` - override the desktop data directory (default: platform-specific Discord cache path)
- `--dry-run` - report what would be imported without writing anything
- `--dry-run` - report what would be imported without creating an archive, importing data, or creating runtime directories
- `--full-cache` - exhaustive Chromium HTTP cache import for historical guild-cache archaeology (slower)
- `--watch-every <duration>` - keep importing on a periodic loop
- `--stats` - attach a full archive coverage snapshot; watched samples after the first include deltas
Expand All @@ -40,6 +40,7 @@ discrawl wiretap --watch-every 10s --stats --json
- scans local `.ldb`, `.log`, `.json`, and `.txt` artifacts for Discord message JSON, plus route-bearing Chromium HTTP cache entries by default
- does not extract, store, or print Discord auth tokens
- persists only compact aggregate import counters for [`coverage`](coverage.html); raw cache paths and payloads are not added to coverage state
- with `--dry-run --stats`, reads coverage from an existing archive without migrating it; reports empty coverage if no archive exists, including in watch mode. Existing archives use normal SQLite read-only access, which may create WAL/SHM sidecar files.

## Default desktop paths

Expand Down
58 changes: 35 additions & 23 deletions internal/cli/admin_commands.go
Original file line number Diff line number Diff line change
Expand Up @@ -403,6 +403,9 @@ func (r *runtime) runWiretap(args []string) error {
if *maxFileBytes <= 0 {
return usageErr(errors.New("--max-file-bytes must be positive"))
}
if *watchEvery > 0 && *watchEvery < time.Second {
return usageErr(errors.New("--watch-every must be at least 1s"))
}
var previousCoverage *store.CoverageReport
runOnce := func(ctx context.Context) error {
stats, err := discorddesktop.Import(ctx, r.store, discorddesktop.Options{
Expand All @@ -416,9 +419,12 @@ func (r *runtime) runWiretap(args []string) error {
return err
}
if *showStats {
coverage, err := r.store.Coverage(ctx, "", r.nowUTC())
if err != nil {
return err
coverage := store.CoverageReport{GeneratedAt: r.nowUTC(), Guilds: []store.CoverageGuild{}}
if r.store != nil {
coverage, err = r.store.Coverage(ctx, "", r.nowUTC())
if err != nil {
return err
}
}
progress := wiretapProgress{Import: stats, Coverage: coverage}
if previousCoverage != nil {
Expand All @@ -430,29 +436,35 @@ func (r *runtime) runWiretap(args []string) error {
}
return r.print(stats)
}
if *watchEvery <= 0 {
return runOnce(r.ctx)
}
if *watchEvery < time.Second {
return usageErr(errors.New("--watch-every must be at least 1s"))
}
ctx, stop := signal.NotifyContext(r.ctx, os.Interrupt, syscall.SIGTERM)
defer stop()
if err := runOnce(ctx); err != nil {
return err
}
ticker := time.NewTicker(*watchEvery)
defer ticker.Stop()
for {
select {
case <-ctx.Done():
return nil
case <-ticker.C:
if err := runOnce(ctx); err != nil {
return err
run := func() error {
if *watchEvery <= 0 {
return runOnce(r.ctx)
}
ctx, stop := signal.NotifyContext(r.ctx, os.Interrupt, syscall.SIGTERM)
defer stop()
if err := runOnce(ctx); err != nil {
return err
}
ticker := time.NewTicker(*watchEvery)
defer ticker.Stop()
for {
select {
case <-ctx.Done():
return nil
case <-ticker.C:
if err := runOnce(ctx); err != nil {
return err
}
}
}
}
if *dryRun {
if *showStats {
return r.withLocalStoreReadOnly(run)
}
return run()
}
return r.withLocalStoreLocked(false, run)
}

func (r *runtime) runStatus(args []string) error {
Expand Down
6 changes: 2 additions & 4 deletions internal/cli/cli.go
Original file line number Diff line number Diff line change
Expand Up @@ -344,10 +344,8 @@ func (r *runtime) dispatch(rest []string) error {
operation = "tail-failure-replay"
}
return r.withServicesLockedOperation(true, operation, func() error { return r.runTail(rest[1:]) })
case "wiretap":
return r.withLocalStoreLocked(false, func() error { return r.runWiretap(rest[1:]) })
case "tap", "cache-import":
return r.withLocalStoreLocked(false, func() error { return r.runWiretap(rest[1:]) })
case "wiretap", "tap", "cache-import":
return r.withConfig(func() error { return r.runWiretap(rest[1:]) })
case "search":
if hasHelpFlag(rest[1:]) {
return printCommandUsage(r.stdout, []string{"search"})
Expand Down
94 changes: 94 additions & 0 deletions internal/cli/wiretap_dry_run_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,94 @@
package cli

import (
"bytes"
"context"
"crypto/sha256"
"encoding/json"
"fmt"
"os"
"path/filepath"
"testing"

"github.com/stretchr/testify/require"

"github.com/openclaw/discrawl/internal/discorddesktop"
)

func dryRunFixture(t *testing.T) (string, string, string) {
t.Helper()
dir := t.TempDir()
db := filepath.Join(dir, "archive", "archive.db")
desktop := filepath.Join(dir, "desktop")
require.NoError(t, os.MkdirAll(desktop, 0o755))
require.NoError(t, os.WriteFile(filepath.Join(desktop, "payload.json"), []byte(`{"id":"111111111111111121","guild_id":"999999999999999996","type":0,"name":"synthetic"}
{"id":"333333333333333346","channel_id":"111111111111111121","content":"Sapphire preview","timestamp":"2026-09-01T12:00:00Z","author":{"id":"222222222222222232","username":"fixture"}}`), 0o600))
cfg := filepath.Join(dir, "config.toml")
require.NoError(t, os.WriteFile(cfg, []byte(fmt.Sprintf("version = 1\ndb_path = %q\ncache_dir = %q\nlog_dir = %q\n", db, filepath.Join(dir, "runtime-cache"), filepath.Join(dir, "runtime-logs"))), 0o600))
return cfg, db, desktop
}

func TestWiretapDryRunDoesNotCreateArchive(t *testing.T) {
for _, command := range []string{"wiretap", "tap", "cache-import"} {
for _, stats := range []bool{false, true} {
t.Run(fmt.Sprintf("%s/stats=%t", command, stats), func(t *testing.T) {
cfg, db, desktop := dryRunFixture(t)
args := []string{"--config", cfg, "--json", command, "--dry-run", "--path", desktop}
if stats {
args = append(args, "--stats")
}
var out bytes.Buffer
require.NoError(t, Run(context.Background(), args, &out, &bytes.Buffer{}))
if stats {
var result wiretapProgress
require.NoError(t, json.Unmarshal(out.Bytes(), &result))
require.Equal(t, 1, result.Import.Messages)
require.Zero(t, result.Coverage.Totals.MessageCount)
require.Empty(t, result.Coverage.Guilds)
} else {
var result discorddesktop.Stats
require.NoError(t, json.Unmarshal(out.Bytes(), &result))
require.Equal(t, 1, result.Messages)
require.True(t, result.DryRun)
}
require.NoDirExists(t, filepath.Dir(db))
require.NoDirExists(t, filepath.Join(filepath.Dir(cfg), "runtime-cache"))
require.NoDirExists(t, filepath.Join(filepath.Dir(cfg), "runtime-logs"))
})
}
}
}

func TestWiretapDryRunReadsExistingCoverageWithoutWriting(t *testing.T) {
cfg, db, desktop := dryRunFixture(t)
args := []string{"--config", cfg, "--json", "wiretap", "--path", desktop}
require.NoError(t, Run(context.Background(), args, &bytes.Buffer{}, &bytes.Buffer{}))
before, err := os.ReadFile(db)
require.NoError(t, err)
var out bytes.Buffer
require.NoError(t, Run(context.Background(), append(args, "--dry-run", "--stats"), &out, &bytes.Buffer{}))
var result wiretapProgress
require.NoError(t, json.Unmarshal(out.Bytes(), &result))
require.Equal(t, 1, result.Coverage.Totals.MessageCount)
after, err := os.ReadFile(db)
require.NoError(t, err)
require.Equal(t, sha256.Sum256(before), sha256.Sum256(after))
}

func TestWiretapExplicitFalseDryRunStillImports(t *testing.T) {
cfg, db, desktop := dryRunFixture(t)
args := []string{"--config", cfg, "wiretap", "--path", desktop, "--dry-run", "--dry-run=false"}
require.NoError(t, Run(context.Background(), args, &bytes.Buffer{}, &bytes.Buffer{}))
require.FileExists(t, db)
}

func TestWiretapDryRunWithoutStatsDoesNotOpenArchive(t *testing.T) {
cfg, db, desktop := dryRunFixture(t)
require.NoError(t, os.MkdirAll(filepath.Dir(db), 0o755))
original := []byte("not a SQLite database")
require.NoError(t, os.WriteFile(db, original, 0o600))
require.NoError(t, Run(context.Background(), []string{"--config", cfg, "wiretap", "--dry-run", "--path", desktop}, &bytes.Buffer{}, &bytes.Buffer{}))
contents, err := os.ReadFile(db)
require.NoError(t, err)
require.Equal(t, original, contents)
}