fix: excluded categories still capture nested threads

docs/configuration.md

@@ -145,6 +145,7 @@ Set `discord.token_source = "keyring"` if you want to require keyring lookup and
 - `guild_ids` is reserved for explicit multi-guild fan-out; usually you do not set this directly
 - `sync.include_category_ids` limits Discord sync and tail collection to the listed categories and all channel/thread descendants; root-level and unrelated channels are skipped
 - `sync.exclude_channel_ids` and `sync.exclude_channel_kinds` apply to historical sync, live tail events, and repair syncs; exclusions always win over category inclusion
+- `sync.exclude_channel_ids` also accepts category IDs: excluding a category or channel excludes all its channel and thread descendants. For opt-out collection, leave `sync.include_category_ids` empty and put unwanted category/channel IDs in `sync.exclude_channel_ids`; new categories are included automatically, subject to Discord access and other exclusions.
 - `sync.exclude_channel_kinds` accepts Discrawl kinds such as `text`, `announcement`, `forum`, `thread_public`, `thread_private`, and `thread_announcement`
 - a non-zero `sync.repair_offset` aligns periodic repairs to local wall-clock boundaries; for example, `repair_every = "6h"` with `repair_offset = "2h"` targets 02:00, 08:00, 14:00, and 20:00 local time
 - `[search.lexical].languages` enables opt-in multilingual FTS fields. Supported presets are Korean (`ko`, Kiwi through `github.com/codingpot/kiwigo`), Japanese (`ja`, Kagome Search through `discrawl-ja`), Chinese (`zh`, GSE CutSearch through `discrawl-zh`), and Arabic (`ar`, in-process light stemming).

internal/syncer/category_exclusion_descendants_test.go

@@ -0,0 +1,103 @@
+package syncer
+
+import (
+	"context"
+	"path/filepath"
+	"testing"
+	"time"
+
+	"github.com/bwmarrin/discordgo"
+	"github.com/stretchr/testify/require"
+
+	"github.com/openclaw/discrawl/internal/store"
+)
+
+func categoryExclusionFixture() []*discordgo.Channel {
+	return []*discordgo.Channel{
+		{ID: "blocked-category", GuildID: "g1", Type: discordgo.ChannelTypeGuildCategory},
+		{ID: "blocked-forum", GuildID: "g1", ParentID: "blocked-category", Type: discordgo.ChannelTypeGuildForum},
+		{ID: "blocked-thread", GuildID: "g1", ParentID: "blocked-forum", Type: discordgo.ChannelTypeGuildPublicThread},
+		{ID: "blocked-text", GuildID: "g1", ParentID: "blocked-category", Type: discordgo.ChannelTypeGuildText},
+		{ID: "blocked-private-thread", GuildID: "g1", ParentID: "blocked-text", Type: discordgo.ChannelTypeGuildPrivateThread},
+		{ID: "new-category", GuildID: "g1", Type: discordgo.ChannelTypeGuildCategory},
+		{ID: "allowed", GuildID: "g1", ParentID: "new-category", Type: discordgo.ChannelTypeGuildText},
+		{ID: "root", GuildID: "g1", Type: discordgo.ChannelTypeGuildText},
+	}
+}
+
+func TestCategoryExclusionFiltersDescendantsWithoutAllowlist(t *testing.T) {
+	t.Parallel()
+	channels := categoryExclusionFixture()
+	scope := newChannelScope([]string{"blocked-category"}, nil, nil)
+	require.Equal(t, []string{"new-category", "allowed", "root"}, channelIDs(filterExcludedDiscordChannels(channels, scope)))
+	// Exclusions still win when the same category is explicitly included.
+	scope = newChannelScope([]string{"blocked-category"}, nil, []string{"blocked-category"})
+	require.Empty(t, filterExcludedDiscordChannels(channels, scope))
+}
+
+func TestCategoryExclusionAppliesToStoredRepairAndTail(t *testing.T) {
+	t.Parallel()
+	ctx := context.Background()
+	s, err := store.Open(ctx, filepath.Join(t.TempDir(), "discrawl.db"))
+	require.NoError(t, err)
+	t.Cleanup(func() { _ = s.Close() })
+	for _, channel := range categoryExclusionFixture() {
+		require.NoError(t, s.UpsertChannel(ctx, toChannelRecord(channel, "{}")))
+	}
+	svc := New(&fakeClient{}, s, nil)
+	svc.SetChannelExclusions([]string{"blocked-category"}, nil)
+	ids := []string{"blocked-thread", "blocked-private-thread", "allowed", "root"}
+	filtered, err := svc.filterExcludedStoredChannelIDs(ctx, "g1", ids, SyncOptions{})
+	require.NoError(t, err)
+	require.Equal(t, []string{"allowed", "root"}, filtered)
+
+	channels, targeted, err := svc.channelList(ctx, "g1", []string{"blocked-thread"}, channelCatalogFull,
+		svc.effectiveChannelExclusions(SyncOptions{}), makeGuildSet([]string{"g1"}), nil, nil)
+	require.NoError(t, err)
+	require.True(t, targeted)
+	require.Empty(t, channels)
+
+	handler := &tailHandler{guilds: makeGuildSet([]string{"g1"}), store: s, exclusions: svc.channelExclusions}
+	require.NoError(t, handler.seedChannelExclusions(ctx))
+	for i, id := range ids {
+		require.NoError(t, handler.OnMessageCreate(ctx, &discordgo.Message{
+			ID: string(rune('1' + i)), GuildID: "g1", ChannelID: id, Content: id,
+			Timestamp: time.Now().UTC(), Author: &discordgo.User{ID: "u1"},
+		}))
+	}
+	messages, err := s.ListMessages(ctx, store.MessageListOptions{GuildIDs: []string{"g1"}, IncludeEmpty: true})
+	require.NoError(t, err)
+	require.Len(t, messages, 2)
+	for _, message := range messages {
+		require.Contains(t, []string{"allowed", "root"}, message.ChannelID)
+	}
+}
+
+func TestTailCategoryExclusionTracksAncestorUpdates(t *testing.T) {
+	t.Parallel()
+	handler := &tailHandler{exclusions: newChannelScope([]string{"blocked-category"}, nil, nil)}
+	thread := &discordgo.Channel{ID: "thread", ParentID: "forum", Type: discordgo.ChannelTypeGuildPublicThread}
+	forum := &discordgo.Channel{ID: "forum", ParentID: "new-category", Type: discordgo.ChannelTypeGuildForum}
+	// Child metadata can arrive before its parent.
+	handler.trackChannelExclusion(thread)
+	handler.trackChannelExclusion(forum)
+	require.False(t, handler.excludeChannel(thread.ID))
+	forum.ParentID = "blocked-category"
+	handler.trackChannelExclusion(forum)
+	require.True(t, handler.excludeChannel(thread.ID))
+	forum.ParentID = "new-category"
+	handler.trackChannelExclusion(forum)
+	require.False(t, handler.excludeChannel(thread.ID))
+}
+
+func TestCategoryExclusionHandlesIncompleteAndCyclicAncestry(t *testing.T) {
+	t.Parallel()
+	scope := newChannelScope([]string{"blocked-category"}, nil, nil)
+	thread := &discordgo.Channel{ID: "thread", ParentID: "forum", Type: discordgo.ChannelTypeGuildPublicThread}
+	forum := &discordgo.Channel{ID: "forum", ParentID: "blocked-category", Type: discordgo.ChannelTypeGuildForum}
+	catalog := map[string]*discordgo.Channel{thread.ID: thread, forum.ID: forum}
+	// The excluded category need not itself be present in the catalog.
+	require.True(t, scope.excludesDiscordChannel(thread, catalog))
+	forum.ParentID = thread.ID
+	require.False(t, scope.excludesDiscordChannel(thread, catalog))
+}

internal/syncer/channel_exclusions.go

@@ -124,36 +124,40 @@ func (e channelExclusions) excludesDiscordChannel(channel *discordgo.Channel, ch
 	if channel == nil {
 		return false
 	}
-	if e.excludesID(channel.ID) || e.excludesKind(channelKind(channel)) {
-		return true
+	for current, seen := channel, map[string]struct{}{}; current != nil; {
+		if e.excludesID(current.ID) || e.excludesKind(channelKind(current)) || e.excludesID(current.ParentID) {
+			return true
+		}
+		if _, ok := seen[current.ID]; ok {
+			break
+		}
+		seen[current.ID] = struct{}{}
+		current = channelByID[current.ParentID]
 	}
 	if channel.ParentID == "" {
 		return !e.allowsUnparentedDiscordChannel(channel)
 	}
-	if e.excludesID(channel.ParentID) {
-		return true
-	}
-	parent := channelByID[channel.ParentID]
-	if parent != nil && e.excludesKind(channelKind(parent)) {
-		return true
-	}
 	return !e.allowsDiscordCategory(channel, channelByID)
 }
 
 func (e channelExclusions) excludesStoredChannel(channel store.ChannelRow, channelByID map[string]store.ChannelRow) bool {
-	if e.excludesID(channel.ID) || e.excludesKind(channel.Kind) {
-		return true
+	for current, seen := channel, map[string]struct{}{}; ; {
+		if e.excludesID(current.ID) || e.excludesKind(current.Kind) || e.excludesID(current.ParentID) {
+			return true
+		}
+		if _, ok := seen[current.ID]; ok {
+			break
+		}
+		seen[current.ID] = struct{}{}
+		parent, ok := channelByID[current.ParentID]
+		if !ok {
+			break
+		}
+		current = parent
 	}
 	if channel.ParentID == "" {
 		return !e.allowsUnparentedStoredChannel(channel)
 	}
-	if e.excludesID(channel.ParentID) {
-		return true
-	}
-	parent, ok := channelByID[channel.ParentID]
-	if ok && e.excludesKind(parent.Kind) {
-		return true
-	}
 	return !e.allowsStoredCategory(channel, channelByID)
 }
 

internal/syncer/channel_exclusions_test.go

@@ -205,11 +205,9 @@ func TestTailAppliesCategoryAndChannelExclusions(t *testing.T) {
 	}
 
 	handler := &tailHandler{
-		guilds:                 makeGuildSet([]string{"g1"}),
-		store:                  s,
-		exclusions:             newChannelScope([]string{"blocked-id"}, []string{"announcement"}, []string{"category-a"}),
-		kindExcludedChannelIDs: map[string]struct{}{},
-		knownChannelIDs:        map[string]struct{}{},
+		guilds:     makeGuildSet([]string{"g1"}),
+		store:      s,
+		exclusions: newChannelScope([]string{"blocked-id"}, []string{"announcement"}, []string{"category-a"}),
 	}
 	require.NoError(t, handler.seedChannelExclusions(ctx))
 

internal/syncer/tail.go

@@ -18,16 +18,14 @@ func (s *Syncer) RunTail(ctx context.Context, guildIDs []string, repairEvery tim
 		return err
 	}
 	handler := &tailHandler{
-		guilds:                 makeGuildSet(guildIDs),
-		store:                  s.store,
-		client:                 s.client,
-		attachmentTextEnabled:  s.attachmentTextEnabled,
-		enqueueEmbeddings:      s.tailEmbeddings,
-		onReady:                s.tailReady,
-		logger:                 s.logger,
-		exclusions:             s.channelExclusions,
-		kindExcludedChannelIDs: map[string]struct{}{},
-		knownChannelIDs:        map[string]struct{}{},
+		guilds:                makeGuildSet(guildIDs),
+		store:                 s.store,
+		client:                s.client,
+		attachmentTextEnabled: s.attachmentTextEnabled,
+		enqueueEmbeddings:     s.tailEmbeddings,
+		onReady:               s.tailReady,
+		logger:                s.logger,
+		exclusions:            s.channelExclusions,
 	}
 	if err := handler.seedChannelExclusions(ctx); err != nil {
 		return fmt.Errorf("seed tail channel exclusions: %w", err)
@@ -205,18 +203,17 @@ func (s *Syncer) logTailRepairResult(result tailRepairResult) {
 }
 
 type tailHandler struct {
-	guilds                 map[string]struct{}
-	store                  *store.Store
-	client                 Client
-	attachmentTextEnabled  bool
-	enqueueEmbeddings      bool
-	failureLedgerTimeout   time.Duration
-	onReady                func(context.Context) error
-	logger                 *slog.Logger
-	exclusions             channelExclusions
-	exclusionMu            sync.RWMutex
-	kindExcludedChannelIDs map[string]struct{}
-	knownChannelIDs        map[string]struct{}
+	guilds                map[string]struct{}
+	store                 *store.Store
+	client                Client
+	attachmentTextEnabled bool
+	enqueueEmbeddings     bool
+	failureLedgerTimeout  time.Duration
+	onReady               func(context.Context) error
+	logger                *slog.Logger
+	exclusions            channelExclusions
+	exclusionMu           sync.RWMutex
+	channelScopeCatalog   map[string]store.ChannelRow
 }
 
 func (t *tailHandler) OnTailReady(ctx context.Context) error {
@@ -556,20 +553,7 @@ func (t *tailHandler) seedChannelExclusions(ctx context.Context) error {
 	}
 	t.exclusionMu.Lock()
 	defer t.exclusionMu.Unlock()
-	if t.kindExcludedChannelIDs == nil {
-		t.kindExcludedChannelIDs = map[string]struct{}{}
-	}
-	if t.knownChannelIDs == nil {
-		t.knownChannelIDs = map[string]struct{}{}
-	}
-	for _, channel := range channels {
-		t.knownChannelIDs[channel.ID] = struct{}{}
-		if t.exclusions.excludesStoredChannel(channel, channelByID) {
-			t.kindExcludedChannelIDs[channel.ID] = struct{}{}
-			continue
-		}
-		delete(t.kindExcludedChannelIDs, channel.ID)
-	}
+	t.channelScopeCatalog = channelByID
 	return nil
 }
 
@@ -579,45 +563,26 @@ func (t *tailHandler) excludeChannel(channelID string) bool {
 	}
 	t.exclusionMu.RLock()
 	defer t.exclusionMu.RUnlock()
-	if _, ok := t.kindExcludedChannelIDs[channelID]; ok {
-		return true
+	if channel, known := t.channelScopeCatalog[channelID]; known {
+		return t.exclusions.excludesStoredChannel(channel, t.channelScopeCatalog)
 	}
-	if !t.exclusions.categoryScopeSet {
-		return false
-	}
-	_, known := t.knownChannelIDs[channelID]
-	return !known
+	return t.exclusions.categoryScopeSet
 }
 
 func (t *tailHandler) trackChannelExclusion(channel *discordgo.Channel) {
 	if channel == nil {
 		return
 	}
-	excluded := t.exclusions.excludesID(channel.ID) || t.exclusions.excludesKind(channelKind(channel))
-	if !excluded && channel.ParentID == "" {
-		excluded = !t.exclusions.allowsUnparentedDiscordChannel(channel)
-	}
-	if !excluded && channel.ParentID != "" {
-		excluded = t.exclusions.excludesID(channel.ParentID)
-		_, allowedCategory := t.exclusions.allowedCategoryIDs[channel.ParentID]
-		if !excluded && (!t.exclusions.categoryScopeSet || !allowedCategory) {
-			excluded = t.excludeChannel(channel.ParentID)
-		}
-	}
 	t.exclusionMu.Lock()
 	defer t.exclusionMu.Unlock()
-	if t.kindExcludedChannelIDs == nil {
-		t.kindExcludedChannelIDs = map[string]struct{}{}
-	}
-	if t.knownChannelIDs == nil {
-		t.knownChannelIDs = map[string]struct{}{}
+	if t.channelScopeCatalog == nil {
+		t.channelScopeCatalog = map[string]store.ChannelRow{}
 	}
-	t.knownChannelIDs[channel.ID] = struct{}{}
-	if excluded {
-		t.kindExcludedChannelIDs[channel.ID] = struct{}{}
-		return
+	// Resolve ancestry when an event is checked, so parent moves and metadata
+	// arriving after a child also update that child's effective scope.
+	t.channelScopeCatalog[channel.ID] = store.ChannelRow{
+		ID: channel.ID, ParentID: channel.ParentID, Kind: channelKind(channel),
 	}
-	delete(t.kindExcludedChannelIDs, channel.ID)
 }
 
 func nextTailRepairDelay(now time.Time, repairEvery, repairOffset time.Duration) time.Duration {