diff --git a/docs/configuration.md b/docs/configuration.md index f4de2e2..2214370 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -145,6 +145,7 @@ Set `discord.token_source = "keyring"` if you want to require keyring lookup and - `guild_ids` is reserved for explicit multi-guild fan-out; usually you do not set this directly - `sync.include_category_ids` limits Discord sync and tail collection to the listed categories and all channel/thread descendants; root-level and unrelated channels are skipped - `sync.exclude_channel_ids` and `sync.exclude_channel_kinds` apply to historical sync, live tail events, and repair syncs; exclusions always win over category inclusion +- `sync.exclude_channel_ids` also accepts category IDs: excluding a category or channel excludes all its channel and thread descendants. For opt-out collection, leave `sync.include_category_ids` empty and put unwanted category/channel IDs in `sync.exclude_channel_ids`; new categories are included automatically, subject to Discord access and other exclusions. - `sync.exclude_channel_kinds` accepts Discrawl kinds such as `text`, `announcement`, `forum`, `thread_public`, `thread_private`, and `thread_announcement` - a non-zero `sync.repair_offset` aligns periodic repairs to local wall-clock boundaries; for example, `repair_every = "6h"` with `repair_offset = "2h"` targets 02:00, 08:00, 14:00, and 20:00 local time - `[search.lexical].languages` enables opt-in multilingual FTS fields. Supported presets are Korean (`ko`, Kiwi through `github.com/codingpot/kiwigo`), Japanese (`ja`, Kagome Search through `discrawl-ja`), Chinese (`zh`, GSE CutSearch through `discrawl-zh`), and Arabic (`ar`, in-process light stemming). diff --git a/internal/syncer/category_exclusion_descendants_test.go b/internal/syncer/category_exclusion_descendants_test.go new file mode 100644 index 0000000..ffd2c1f --- /dev/null +++ b/internal/syncer/category_exclusion_descendants_test.go @@ -0,0 +1,103 @@ +package syncer + +import ( + "context" + "path/filepath" + "testing" + "time" + + "github.com/bwmarrin/discordgo" + "github.com/stretchr/testify/require" + + "github.com/openclaw/discrawl/internal/store" +) + +func categoryExclusionFixture() []*discordgo.Channel { + return []*discordgo.Channel{ + {ID: "blocked-category", GuildID: "g1", Type: discordgo.ChannelTypeGuildCategory}, + {ID: "blocked-forum", GuildID: "g1", ParentID: "blocked-category", Type: discordgo.ChannelTypeGuildForum}, + {ID: "blocked-thread", GuildID: "g1", ParentID: "blocked-forum", Type: discordgo.ChannelTypeGuildPublicThread}, + {ID: "blocked-text", GuildID: "g1", ParentID: "blocked-category", Type: discordgo.ChannelTypeGuildText}, + {ID: "blocked-private-thread", GuildID: "g1", ParentID: "blocked-text", Type: discordgo.ChannelTypeGuildPrivateThread}, + {ID: "new-category", GuildID: "g1", Type: discordgo.ChannelTypeGuildCategory}, + {ID: "allowed", GuildID: "g1", ParentID: "new-category", Type: discordgo.ChannelTypeGuildText}, + {ID: "root", GuildID: "g1", Type: discordgo.ChannelTypeGuildText}, + } +} + +func TestCategoryExclusionFiltersDescendantsWithoutAllowlist(t *testing.T) { + t.Parallel() + channels := categoryExclusionFixture() + scope := newChannelScope([]string{"blocked-category"}, nil, nil) + require.Equal(t, []string{"new-category", "allowed", "root"}, channelIDs(filterExcludedDiscordChannels(channels, scope))) + // Exclusions still win when the same category is explicitly included. + scope = newChannelScope([]string{"blocked-category"}, nil, []string{"blocked-category"}) + require.Empty(t, filterExcludedDiscordChannels(channels, scope)) +} + +func TestCategoryExclusionAppliesToStoredRepairAndTail(t *testing.T) { + t.Parallel() + ctx := context.Background() + s, err := store.Open(ctx, filepath.Join(t.TempDir(), "discrawl.db")) + require.NoError(t, err) + t.Cleanup(func() { _ = s.Close() }) + for _, channel := range categoryExclusionFixture() { + require.NoError(t, s.UpsertChannel(ctx, toChannelRecord(channel, "{}"))) + } + svc := New(&fakeClient{}, s, nil) + svc.SetChannelExclusions([]string{"blocked-category"}, nil) + ids := []string{"blocked-thread", "blocked-private-thread", "allowed", "root"} + filtered, err := svc.filterExcludedStoredChannelIDs(ctx, "g1", ids, SyncOptions{}) + require.NoError(t, err) + require.Equal(t, []string{"allowed", "root"}, filtered) + + channels, targeted, err := svc.channelList(ctx, "g1", []string{"blocked-thread"}, channelCatalogFull, + svc.effectiveChannelExclusions(SyncOptions{}), makeGuildSet([]string{"g1"}), nil, nil) + require.NoError(t, err) + require.True(t, targeted) + require.Empty(t, channels) + + handler := &tailHandler{guilds: makeGuildSet([]string{"g1"}), store: s, exclusions: svc.channelExclusions} + require.NoError(t, handler.seedChannelExclusions(ctx)) + for i, id := range ids { + require.NoError(t, handler.OnMessageCreate(ctx, &discordgo.Message{ + ID: string(rune('1' + i)), GuildID: "g1", ChannelID: id, Content: id, + Timestamp: time.Now().UTC(), Author: &discordgo.User{ID: "u1"}, + })) + } + messages, err := s.ListMessages(ctx, store.MessageListOptions{GuildIDs: []string{"g1"}, IncludeEmpty: true}) + require.NoError(t, err) + require.Len(t, messages, 2) + for _, message := range messages { + require.Contains(t, []string{"allowed", "root"}, message.ChannelID) + } +} + +func TestTailCategoryExclusionTracksAncestorUpdates(t *testing.T) { + t.Parallel() + handler := &tailHandler{exclusions: newChannelScope([]string{"blocked-category"}, nil, nil)} + thread := &discordgo.Channel{ID: "thread", ParentID: "forum", Type: discordgo.ChannelTypeGuildPublicThread} + forum := &discordgo.Channel{ID: "forum", ParentID: "new-category", Type: discordgo.ChannelTypeGuildForum} + // Child metadata can arrive before its parent. + handler.trackChannelExclusion(thread) + handler.trackChannelExclusion(forum) + require.False(t, handler.excludeChannel(thread.ID)) + forum.ParentID = "blocked-category" + handler.trackChannelExclusion(forum) + require.True(t, handler.excludeChannel(thread.ID)) + forum.ParentID = "new-category" + handler.trackChannelExclusion(forum) + require.False(t, handler.excludeChannel(thread.ID)) +} + +func TestCategoryExclusionHandlesIncompleteAndCyclicAncestry(t *testing.T) { + t.Parallel() + scope := newChannelScope([]string{"blocked-category"}, nil, nil) + thread := &discordgo.Channel{ID: "thread", ParentID: "forum", Type: discordgo.ChannelTypeGuildPublicThread} + forum := &discordgo.Channel{ID: "forum", ParentID: "blocked-category", Type: discordgo.ChannelTypeGuildForum} + catalog := map[string]*discordgo.Channel{thread.ID: thread, forum.ID: forum} + // The excluded category need not itself be present in the catalog. + require.True(t, scope.excludesDiscordChannel(thread, catalog)) + forum.ParentID = thread.ID + require.False(t, scope.excludesDiscordChannel(thread, catalog)) +} diff --git a/internal/syncer/channel_exclusions.go b/internal/syncer/channel_exclusions.go index c5188ba..1213087 100644 --- a/internal/syncer/channel_exclusions.go +++ b/internal/syncer/channel_exclusions.go @@ -124,36 +124,40 @@ func (e channelExclusions) excludesDiscordChannel(channel *discordgo.Channel, ch if channel == nil { return false } - if e.excludesID(channel.ID) || e.excludesKind(channelKind(channel)) { - return true + for current, seen := channel, map[string]struct{}{}; current != nil; { + if e.excludesID(current.ID) || e.excludesKind(channelKind(current)) || e.excludesID(current.ParentID) { + return true + } + if _, ok := seen[current.ID]; ok { + break + } + seen[current.ID] = struct{}{} + current = channelByID[current.ParentID] } if channel.ParentID == "" { return !e.allowsUnparentedDiscordChannel(channel) } - if e.excludesID(channel.ParentID) { - return true - } - parent := channelByID[channel.ParentID] - if parent != nil && e.excludesKind(channelKind(parent)) { - return true - } return !e.allowsDiscordCategory(channel, channelByID) } func (e channelExclusions) excludesStoredChannel(channel store.ChannelRow, channelByID map[string]store.ChannelRow) bool { - if e.excludesID(channel.ID) || e.excludesKind(channel.Kind) { - return true + for current, seen := channel, map[string]struct{}{}; ; { + if e.excludesID(current.ID) || e.excludesKind(current.Kind) || e.excludesID(current.ParentID) { + return true + } + if _, ok := seen[current.ID]; ok { + break + } + seen[current.ID] = struct{}{} + parent, ok := channelByID[current.ParentID] + if !ok { + break + } + current = parent } if channel.ParentID == "" { return !e.allowsUnparentedStoredChannel(channel) } - if e.excludesID(channel.ParentID) { - return true - } - parent, ok := channelByID[channel.ParentID] - if ok && e.excludesKind(parent.Kind) { - return true - } return !e.allowsStoredCategory(channel, channelByID) } diff --git a/internal/syncer/channel_exclusions_test.go b/internal/syncer/channel_exclusions_test.go index a188a3e..b1fee4f 100644 --- a/internal/syncer/channel_exclusions_test.go +++ b/internal/syncer/channel_exclusions_test.go @@ -205,11 +205,9 @@ func TestTailAppliesCategoryAndChannelExclusions(t *testing.T) { } handler := &tailHandler{ - guilds: makeGuildSet([]string{"g1"}), - store: s, - exclusions: newChannelScope([]string{"blocked-id"}, []string{"announcement"}, []string{"category-a"}), - kindExcludedChannelIDs: map[string]struct{}{}, - knownChannelIDs: map[string]struct{}{}, + guilds: makeGuildSet([]string{"g1"}), + store: s, + exclusions: newChannelScope([]string{"blocked-id"}, []string{"announcement"}, []string{"category-a"}), } require.NoError(t, handler.seedChannelExclusions(ctx)) diff --git a/internal/syncer/tail.go b/internal/syncer/tail.go index 3b2f6b9..6e739c7 100644 --- a/internal/syncer/tail.go +++ b/internal/syncer/tail.go @@ -18,16 +18,14 @@ func (s *Syncer) RunTail(ctx context.Context, guildIDs []string, repairEvery tim return err } handler := &tailHandler{ - guilds: makeGuildSet(guildIDs), - store: s.store, - client: s.client, - attachmentTextEnabled: s.attachmentTextEnabled, - enqueueEmbeddings: s.tailEmbeddings, - onReady: s.tailReady, - logger: s.logger, - exclusions: s.channelExclusions, - kindExcludedChannelIDs: map[string]struct{}{}, - knownChannelIDs: map[string]struct{}{}, + guilds: makeGuildSet(guildIDs), + store: s.store, + client: s.client, + attachmentTextEnabled: s.attachmentTextEnabled, + enqueueEmbeddings: s.tailEmbeddings, + onReady: s.tailReady, + logger: s.logger, + exclusions: s.channelExclusions, } if err := handler.seedChannelExclusions(ctx); err != nil { return fmt.Errorf("seed tail channel exclusions: %w", err) @@ -205,18 +203,17 @@ func (s *Syncer) logTailRepairResult(result tailRepairResult) { } type tailHandler struct { - guilds map[string]struct{} - store *store.Store - client Client - attachmentTextEnabled bool - enqueueEmbeddings bool - failureLedgerTimeout time.Duration - onReady func(context.Context) error - logger *slog.Logger - exclusions channelExclusions - exclusionMu sync.RWMutex - kindExcludedChannelIDs map[string]struct{} - knownChannelIDs map[string]struct{} + guilds map[string]struct{} + store *store.Store + client Client + attachmentTextEnabled bool + enqueueEmbeddings bool + failureLedgerTimeout time.Duration + onReady func(context.Context) error + logger *slog.Logger + exclusions channelExclusions + exclusionMu sync.RWMutex + channelScopeCatalog map[string]store.ChannelRow } func (t *tailHandler) OnTailReady(ctx context.Context) error { @@ -556,20 +553,7 @@ func (t *tailHandler) seedChannelExclusions(ctx context.Context) error { } t.exclusionMu.Lock() defer t.exclusionMu.Unlock() - if t.kindExcludedChannelIDs == nil { - t.kindExcludedChannelIDs = map[string]struct{}{} - } - if t.knownChannelIDs == nil { - t.knownChannelIDs = map[string]struct{}{} - } - for _, channel := range channels { - t.knownChannelIDs[channel.ID] = struct{}{} - if t.exclusions.excludesStoredChannel(channel, channelByID) { - t.kindExcludedChannelIDs[channel.ID] = struct{}{} - continue - } - delete(t.kindExcludedChannelIDs, channel.ID) - } + t.channelScopeCatalog = channelByID return nil } @@ -579,45 +563,26 @@ func (t *tailHandler) excludeChannel(channelID string) bool { } t.exclusionMu.RLock() defer t.exclusionMu.RUnlock() - if _, ok := t.kindExcludedChannelIDs[channelID]; ok { - return true + if channel, known := t.channelScopeCatalog[channelID]; known { + return t.exclusions.excludesStoredChannel(channel, t.channelScopeCatalog) } - if !t.exclusions.categoryScopeSet { - return false - } - _, known := t.knownChannelIDs[channelID] - return !known + return t.exclusions.categoryScopeSet } func (t *tailHandler) trackChannelExclusion(channel *discordgo.Channel) { if channel == nil { return } - excluded := t.exclusions.excludesID(channel.ID) || t.exclusions.excludesKind(channelKind(channel)) - if !excluded && channel.ParentID == "" { - excluded = !t.exclusions.allowsUnparentedDiscordChannel(channel) - } - if !excluded && channel.ParentID != "" { - excluded = t.exclusions.excludesID(channel.ParentID) - _, allowedCategory := t.exclusions.allowedCategoryIDs[channel.ParentID] - if !excluded && (!t.exclusions.categoryScopeSet || !allowedCategory) { - excluded = t.excludeChannel(channel.ParentID) - } - } t.exclusionMu.Lock() defer t.exclusionMu.Unlock() - if t.kindExcludedChannelIDs == nil { - t.kindExcludedChannelIDs = map[string]struct{}{} - } - if t.knownChannelIDs == nil { - t.knownChannelIDs = map[string]struct{}{} + if t.channelScopeCatalog == nil { + t.channelScopeCatalog = map[string]store.ChannelRow{} } - t.knownChannelIDs[channel.ID] = struct{}{} - if excluded { - t.kindExcludedChannelIDs[channel.ID] = struct{}{} - return + // Resolve ancestry when an event is checked, so parent moves and metadata + // arriving after a child also update that child's effective scope. + t.channelScopeCatalog[channel.ID] = store.ChannelRow{ + ID: channel.ID, ParentID: channel.ParentID, Kind: channelKind(channel), } - delete(t.kindExcludedChannelIDs, channel.ID) } func nextTailRepairDelay(now time.Time, repairEvery, repairOffset time.Duration) time.Duration {