From 59c427c89669f54455e00b72c5f7a7fe6bfa3f78 Mon Sep 17 00:00:00 2001 From: Arnab Nandy Date: Fri, 3 Jul 2026 01:01:14 +0530 Subject: [PATCH] Restrict tech searches to matching repositories --- src/features/issues/data/search-options.ts | 24 ++-- src/features/issues/server/github-search.ts | 127 ++++++++++++++++-- src/features/issues/types/search.ts | 5 + .../issues/server/github-search.test.ts | 62 ++++++++- 4 files changed, 193 insertions(+), 25 deletions(-) diff --git a/src/features/issues/data/search-options.ts b/src/features/issues/data/search-options.ts index 9400009..da9ba49 100644 --- a/src/features/issues/data/search-options.ts +++ b/src/features/issues/data/search-options.ts @@ -31,7 +31,6 @@ export const GITHUB_LABELS: Record = { }; export const LANGUAGE_ALIASES: Record = { - angular: "TypeScript", csharp: "C#", "c#": "C#", cpp: "C++", @@ -40,18 +39,25 @@ export const LANGUAGE_ALIASES: Record = { golang: "Go", java: "Java", javascript: "JavaScript", - next: "TypeScript", - "next.js": "TypeScript", - nextjs: "TypeScript", - node: "JavaScript", - "node.js": "JavaScript", - nodejs: "JavaScript", python: "Python", - react: "TypeScript", ruby: "Ruby", rust: "Rust", typescript: "TypeScript", - vue: "TypeScript", +}; + +export const TOPIC_ALIASES: Record = { + angular: { topic: "angular", language: "TypeScript" }, + kubernetes: { topic: "kubernetes", language: "Go" }, + next: { topic: "nextjs", language: "TypeScript" }, + "next.js": { topic: "nextjs", language: "TypeScript" }, + nextjs: { topic: "nextjs", language: "TypeScript" }, + node: { topic: "nodejs", language: "JavaScript" }, + "node.js": { topic: "nodejs", language: "JavaScript" }, + nodejs: { topic: "nodejs", language: "JavaScript" }, + react: { topic: "react", language: "TypeScript" }, + "spring boot": { topic: "spring-boot", language: "Java" }, + springboot: { topic: "spring-boot", language: "Java" }, + vue: { topic: "vue", language: "TypeScript" }, }; export const SORT_OPTIONS = [ diff --git a/src/features/issues/server/github-search.ts b/src/features/issues/server/github-search.ts index 05901d6..b6b3980 100644 --- a/src/features/issues/server/github-search.ts +++ b/src/features/issues/server/github-search.ts @@ -4,11 +4,13 @@ import { HACKTOBERFEST_FILTERS, LINKED_PR_FILTERS, LANGUAGE_ALIASES, + TOPIC_ALIASES, } from "@/features/issues/data/search-options"; import { rankIssues } from "@/features/issues/lib/ranking"; import type { GitHubIssue, GitHubRepo, + GitHubRepoSearchResponse, GitHubSearchResponse, GitHubTimelineEvent, Issue, @@ -18,6 +20,8 @@ import type { const PAGE_SIZE = 24; const CANDIDATE_PAGE_COUNT = 5; +const REPO_SEARCH_PAGE_SIZE = 20; +const REPO_ISSUE_BATCH_SIZE = 10; function normalize(value: string | null) { return (value ?? "").trim().toLowerCase(); @@ -38,6 +42,38 @@ function buildTechQualifier(tech: string) { return quoteSearchValue(tech.trim()); } +function buildRepoTopicQuery(tech: string) { + const normalized = normalize(tech); + const topicAlias = TOPIC_ALIASES[normalized]; + const language = LANGUAGE_ALIASES[normalized]; + + if (language && !topicAlias) { + return null; + } + + const topic = topicAlias?.topic ?? normalized.replaceAll(/\s+/g, "-"); + const queryParts = [ + `topic:${quoteSearchValue(topic)}`, + "archived:false", + ]; + + if (topicAlias?.language) { + queryParts.push(`language:${quoteSearchValue(topicAlias.language)}`); + } + + return queryParts.join(" "); +} + +function buildRepoScopeQualifier(repoNames: string[]) { + const qualifiers = repoNames.map((repoName) => `repo:${repoName}`); + + if (qualifiers.length === 1) { + return qualifiers[0]; + } + + return `(${qualifiers.join(" OR ")})`; +} + function buildLinkedPrQualifier(linkedPr: string) { if (linkedPr === "yes") { return "linked:pr"; @@ -204,30 +240,90 @@ export async function searchGitHubIssues({ const hacktoberfest = HACKTOBERFEST_FILTERS.has(rawHacktoberfest ?? "") ? rawHacktoberfest! : "any"; + const token = process.env.GITHUB_TOKEN; + const repoTopicQuery = buildRepoTopicQuery(tech); + let matchingRepos: GitHubRepo[] = []; const queryParts = [ "is:issue", "is:open", "archived:false", - buildTechQualifier(tech), - `label:${quoteSearchValue(label)}`, ]; const linkedPrQualifier = buildLinkedPrQualifier(linkedPr); + if (repoTopicQuery) { + const repoSearchUrl = new URL("https://api.github.com/search/repositories"); + repoSearchUrl.searchParams.set("q", repoTopicQuery); + repoSearchUrl.searchParams.set("sort", "updated"); + repoSearchUrl.searchParams.set("order", "desc"); + repoSearchUrl.searchParams.set("per_page", String(REPO_SEARCH_PAGE_SIZE)); + repoSearchUrl.searchParams.set("page", "1"); + + const repoSearchResult = await githubFetch( + repoSearchUrl.toString(), + token, + 7200, + ); + matchingRepos = repoSearchResult.data.items; + } else { + queryParts.push(buildTechQualifier(tech)); + } + + queryParts.push(`label:${quoteSearchValue(label)}`); + if (linkedPrQualifier) { queryParts.push(linkedPrQualifier); } - const query = queryParts.join(" "); + const repoBatches = + repoTopicQuery && matchingRepos.length > 0 + ? Array.from( + { length: Math.ceil(matchingRepos.length / REPO_ISSUE_BATCH_SIZE) }, + (_, index) => + matchingRepos.slice( + index * REPO_ISSUE_BATCH_SIZE, + (index + 1) * REPO_ISSUE_BATCH_SIZE, + ), + ) + : []; + + if (repoTopicQuery && repoBatches.length === 0) { + return { + query: repoTopicQuery, + totalCount: 0, + candidateCount: 0, + rateLimitRemaining: null, + tokenConfigured: Boolean(token), + issues: [], + page, + }; + } - const token = process.env.GITHUB_TOKEN; - const searchUrls = Array.from({ length: CANDIDATE_PAGE_COUNT }, (_, index) => { - const url = new URL("https://api.github.com/search/issues"); - url.searchParams.set("q", query); - url.searchParams.set("sort", sort); - url.searchParams.set("order", "desc"); - url.searchParams.set("per_page", String(PAGE_SIZE)); - url.searchParams.set("page", String(index + 1)); - return url.toString(); + const issueQueries = + repoBatches.length > 0 + ? repoBatches.slice(0, CANDIDATE_PAGE_COUNT).map((repoBatch) => + [ + ...queryParts, + buildRepoScopeQualifier(repoBatch.map((repo) => repo.full_name)), + ].join(" "), + ) + : [queryParts.join(" ")]; + const query = issueQueries.join(" | "); + + const searchUrls = issueQueries.flatMap((issueQuery) => { + const pageNumbers = + repoBatches.length > 0 + ? [1] + : Array.from({ length: CANDIDATE_PAGE_COUNT }, (_, index) => index + 1); + + return pageNumbers.map((pageNumber) => { + const url = new URL("https://api.github.com/search/issues"); + url.searchParams.set("q", issueQuery); + url.searchParams.set("sort", sort); + url.searchParams.set("order", "desc"); + url.searchParams.set("per_page", String(PAGE_SIZE)); + url.searchParams.set("page", String(pageNumber)); + return url.toString(); + }); }); const searchResults = await Promise.all( searchUrls.map((url) => githubFetch(url, token, 180)), @@ -235,14 +331,16 @@ export async function searchGitHubIssues({ const totalCount = searchResults[0]?.data.total_count ?? 0; const rateLimitRemaining = searchResults.at(-1)?.rateLimitRemaining ?? null; const candidateIssues = dedupeIssues(searchResults.flatMap((result) => result.data.items)); + const repoEntriesFromSearch = matchingRepos.map((repo) => [repo.full_name, repo] as const); + const repoEntriesFromSearchMap = new Map(repoEntriesFromSearch); const shouldFetchRepos = Boolean(token) || hacktoberfest === "only"; const repoNames = shouldFetchRepos ? Array.from( new Set(candidateIssues.map((item) => getRepoFullName(item.repository_url))), - ) + ).filter((fullName) => !repoEntriesFromSearchMap.has(fullName)) : []; - const repoEntries = await Promise.all( + const fetchedRepoEntries = await Promise.all( repoNames.map(async (fullName) => { try { const repo = await githubFetch( @@ -256,6 +354,7 @@ export async function searchGitHubIssues({ } }), ); + const repoEntries = [...repoEntriesFromSearch, ...fetchedRepoEntries]; const commentEntries = await Promise.all( candidateIssues.map(async (issue) => { diff --git a/src/features/issues/types/search.ts b/src/features/issues/types/search.ts index cbe4340..ccfa554 100644 --- a/src/features/issues/types/search.ts +++ b/src/features/issues/types/search.ts @@ -58,6 +58,11 @@ export type GitHubSearchResponse = { items: GitHubIssue[]; }; +export type GitHubRepoSearchResponse = { + total_count: number; + items: GitHubRepo[]; +}; + export type GitHubRepo = { full_name: string; html_url: string; diff --git a/tests/features/issues/server/github-search.test.ts b/tests/features/issues/server/github-search.test.ts index bef49c5..dc5a8cc 100644 --- a/tests/features/issues/server/github-search.test.ts +++ b/tests/features/issues/server/github-search.test.ts @@ -106,7 +106,7 @@ describe("searchGitHubIssues", () => { vi.stubGlobal("fetch", fetchMock); const result = await searchGitHubIssues({ - tech: "React", + tech: "TypeScript", label: "good-first-issue", sort: "updated", linkedPr: "yes", @@ -125,6 +125,64 @@ describe("searchGitHubIssues", () => { expect(result.candidateCount).toBe(1); }); + it("searches framework terms by repository topic instead of issue text", async () => { + const springIssue = githubIssue({ + html_url: "https://github.com/spring-projects/spring-boot/issues/123", + repository_url: "https://api.github.com/repos/spring-projects/spring-boot", + title: "Improve actuator diagnostics", + labels: [{ name: "help wanted" }], + }); + const fetchMock = vi + .fn() + .mockResolvedValueOnce( + jsonResponse({ + total_count: 1, + items: [ + { + full_name: "spring-projects/spring-boot", + html_url: "https://github.com/spring-projects/spring-boot", + stargazers_count: 82000, + archived: false, + topics: ["spring-boot"], + }, + ], + }), + ) + .mockResolvedValueOnce( + jsonResponse({ + total_count: 1, + items: [springIssue], + }), + ); + + vi.stubGlobal("fetch", fetchMock); + + const result = await searchGitHubIssues({ + tech: "Spring Boot", + label: "help-wanted", + sort: "updated", + linkedPr: "any", + }); + + const repoSearchUrl = new URL(fetchMock.mock.calls[0][0] as string); + const issueSearchUrl = new URL(fetchMock.mock.calls[1][0] as string); + const issueQuery = issueSearchUrl.searchParams.get("q") ?? ""; + + expect(repoSearchUrl.pathname).toBe("/search/repositories"); + expect(repoSearchUrl.searchParams.get("q")).toBe( + "topic:spring-boot archived:false language:Java", + ); + expect(issueSearchUrl.pathname).toBe("/search/issues"); + expect(issueQuery).toBe( + "is:issue is:open archived:false label:\"help wanted\" repo:spring-projects/spring-boot", + ); + expect(issueQuery).not.toContain("Spring Boot"); + expect(result.issues[0]).toMatchObject({ + repo: "spring-projects/spring-boot", + stars: 82000, + }); + }); + it("adds the negative linked PR qualifier for no-linked-PR searches", async () => { const fetchMock = vi.fn(); searchPageResponses([githubIssue()]).forEach((response) => { @@ -332,7 +390,7 @@ describe("searchGitHubIssues", () => { vi.stubGlobal("fetch", fetchMock); const result = await searchGitHubIssues({ - tech: "React", + tech: "TypeScript", label: "good-first-issue", sort: "updated", linkedPr: "any",