Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
24 changes: 15 additions & 9 deletions src/features/issues/data/search-options.ts
Original file line number Diff line number Diff line change
Expand Up @@ -31,7 +31,6 @@ export const GITHUB_LABELS: Record<string, string> = {
};

export const LANGUAGE_ALIASES: Record<string, string> = {
angular: "TypeScript",
csharp: "C#",
"c#": "C#",
cpp: "C++",
Expand All @@ -40,18 +39,25 @@ export const LANGUAGE_ALIASES: Record<string, string> = {
golang: "Go",
java: "Java",
javascript: "JavaScript",
next: "TypeScript",
"next.js": "TypeScript",
nextjs: "TypeScript",
node: "JavaScript",
"node.js": "JavaScript",
nodejs: "JavaScript",
python: "Python",
react: "TypeScript",
ruby: "Ruby",
rust: "Rust",
typescript: "TypeScript",
vue: "TypeScript",
};

export const TOPIC_ALIASES: Record<string, { topic: string; language?: string }> = {
angular: { topic: "angular", language: "TypeScript" },
kubernetes: { topic: "kubernetes", language: "Go" },
next: { topic: "nextjs", language: "TypeScript" },
"next.js": { topic: "nextjs", language: "TypeScript" },
nextjs: { topic: "nextjs", language: "TypeScript" },
node: { topic: "nodejs", language: "JavaScript" },
"node.js": { topic: "nodejs", language: "JavaScript" },
nodejs: { topic: "nodejs", language: "JavaScript" },
react: { topic: "react", language: "TypeScript" },
"spring boot": { topic: "spring-boot", language: "Java" },
springboot: { topic: "spring-boot", language: "Java" },
vue: { topic: "vue", language: "TypeScript" },
};

export const SORT_OPTIONS = [
Expand Down
127 changes: 113 additions & 14 deletions src/features/issues/server/github-search.ts
Original file line number Diff line number Diff line change
Expand Up @@ -4,11 +4,13 @@ import {
HACKTOBERFEST_FILTERS,
LINKED_PR_FILTERS,
LANGUAGE_ALIASES,
TOPIC_ALIASES,
} from "@/features/issues/data/search-options";
import { rankIssues } from "@/features/issues/lib/ranking";
import type {
GitHubIssue,
GitHubRepo,
GitHubRepoSearchResponse,
GitHubSearchResponse,
GitHubTimelineEvent,
Issue,
Expand All @@ -18,6 +20,8 @@ import type {

const PAGE_SIZE = 24;
const CANDIDATE_PAGE_COUNT = 5;
const REPO_SEARCH_PAGE_SIZE = 20;
const REPO_ISSUE_BATCH_SIZE = 10;

function normalize(value: string | null) {
return (value ?? "").trim().toLowerCase();
Expand All @@ -38,6 +42,38 @@ function buildTechQualifier(tech: string) {
return quoteSearchValue(tech.trim());
}

function buildRepoTopicQuery(tech: string) {
const normalized = normalize(tech);
const topicAlias = TOPIC_ALIASES[normalized];
const language = LANGUAGE_ALIASES[normalized];

if (language && !topicAlias) {
return null;
}

const topic = topicAlias?.topic ?? normalized.replaceAll(/\s+/g, "-");
const queryParts = [
`topic:${quoteSearchValue(topic)}`,
"archived:false",
];

if (topicAlias?.language) {
queryParts.push(`language:${quoteSearchValue(topicAlias.language)}`);
}

return queryParts.join(" ");
}

function buildRepoScopeQualifier(repoNames: string[]) {
const qualifiers = repoNames.map((repoName) => `repo:${repoName}`);

if (qualifiers.length === 1) {
return qualifiers[0];
}

return `(${qualifiers.join(" OR ")})`;
}

function buildLinkedPrQualifier(linkedPr: string) {
if (linkedPr === "yes") {
return "linked:pr";
Expand Down Expand Up @@ -204,45 +240,107 @@ export async function searchGitHubIssues({
const hacktoberfest = HACKTOBERFEST_FILTERS.has(rawHacktoberfest ?? "")
? rawHacktoberfest!
: "any";
const token = process.env.GITHUB_TOKEN;
const repoTopicQuery = buildRepoTopicQuery(tech);
let matchingRepos: GitHubRepo[] = [];
const queryParts = [
"is:issue",
"is:open",
"archived:false",
buildTechQualifier(tech),
`label:${quoteSearchValue(label)}`,
];
const linkedPrQualifier = buildLinkedPrQualifier(linkedPr);

if (repoTopicQuery) {
const repoSearchUrl = new URL("https://api.github.com/search/repositories");
repoSearchUrl.searchParams.set("q", repoTopicQuery);
repoSearchUrl.searchParams.set("sort", "updated");
repoSearchUrl.searchParams.set("order", "desc");
repoSearchUrl.searchParams.set("per_page", String(REPO_SEARCH_PAGE_SIZE));
repoSearchUrl.searchParams.set("page", "1");

const repoSearchResult = await githubFetch<GitHubRepoSearchResponse>(
repoSearchUrl.toString(),
token,
7200,
);
matchingRepos = repoSearchResult.data.items;
} else {
queryParts.push(buildTechQualifier(tech));
}

queryParts.push(`label:${quoteSearchValue(label)}`);

if (linkedPrQualifier) {
queryParts.push(linkedPrQualifier);
}

const query = queryParts.join(" ");
const repoBatches =
repoTopicQuery && matchingRepos.length > 0
? Array.from(
{ length: Math.ceil(matchingRepos.length / REPO_ISSUE_BATCH_SIZE) },
(_, index) =>
matchingRepos.slice(
index * REPO_ISSUE_BATCH_SIZE,
(index + 1) * REPO_ISSUE_BATCH_SIZE,
),
)
: [];

if (repoTopicQuery && repoBatches.length === 0) {
return {
query: repoTopicQuery,
totalCount: 0,
candidateCount: 0,
rateLimitRemaining: null,
tokenConfigured: Boolean(token),
issues: [],
page,
};
}

const token = process.env.GITHUB_TOKEN;
const searchUrls = Array.from({ length: CANDIDATE_PAGE_COUNT }, (_, index) => {
const url = new URL("https://api.github.com/search/issues");
url.searchParams.set("q", query);
url.searchParams.set("sort", sort);
url.searchParams.set("order", "desc");
url.searchParams.set("per_page", String(PAGE_SIZE));
url.searchParams.set("page", String(index + 1));
return url.toString();
const issueQueries =
repoBatches.length > 0
? repoBatches.slice(0, CANDIDATE_PAGE_COUNT).map((repoBatch) =>
[
...queryParts,
buildRepoScopeQualifier(repoBatch.map((repo) => repo.full_name)),
].join(" "),
)
: [queryParts.join(" ")];
const query = issueQueries.join(" | ");

const searchUrls = issueQueries.flatMap((issueQuery) => {
const pageNumbers =
repoBatches.length > 0
? [1]
: Array.from({ length: CANDIDATE_PAGE_COUNT }, (_, index) => index + 1);

return pageNumbers.map((pageNumber) => {
const url = new URL("https://api.github.com/search/issues");
url.searchParams.set("q", issueQuery);
url.searchParams.set("sort", sort);
url.searchParams.set("order", "desc");
url.searchParams.set("per_page", String(PAGE_SIZE));
url.searchParams.set("page", String(pageNumber));
return url.toString();
});
});
const searchResults = await Promise.all(
searchUrls.map((url) => githubFetch<GitHubSearchResponse>(url, token, 180)),
);
const totalCount = searchResults[0]?.data.total_count ?? 0;
const rateLimitRemaining = searchResults.at(-1)?.rateLimitRemaining ?? null;
const candidateIssues = dedupeIssues(searchResults.flatMap((result) => result.data.items));
const repoEntriesFromSearch = matchingRepos.map((repo) => [repo.full_name, repo] as const);
const repoEntriesFromSearchMap = new Map(repoEntriesFromSearch);
const shouldFetchRepos = Boolean(token) || hacktoberfest === "only";
const repoNames = shouldFetchRepos
? Array.from(
new Set(candidateIssues.map((item) => getRepoFullName(item.repository_url))),
)
).filter((fullName) => !repoEntriesFromSearchMap.has(fullName))
: [];

const repoEntries = await Promise.all(
const fetchedRepoEntries = await Promise.all(
repoNames.map(async (fullName) => {
try {
const repo = await githubFetch<GitHubRepo>(
Expand All @@ -256,6 +354,7 @@ export async function searchGitHubIssues({
}
}),
);
const repoEntries = [...repoEntriesFromSearch, ...fetchedRepoEntries];

const commentEntries = await Promise.all(
candidateIssues.map(async (issue) => {
Expand Down
5 changes: 5 additions & 0 deletions src/features/issues/types/search.ts
Original file line number Diff line number Diff line change
Expand Up @@ -58,6 +58,11 @@ export type GitHubSearchResponse = {
items: GitHubIssue[];
};

export type GitHubRepoSearchResponse = {
total_count: number;
items: GitHubRepo[];
};

export type GitHubRepo = {
full_name: string;
html_url: string;
Expand Down
62 changes: 60 additions & 2 deletions tests/features/issues/server/github-search.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -106,7 +106,7 @@ describe("searchGitHubIssues", () => {
vi.stubGlobal("fetch", fetchMock);

const result = await searchGitHubIssues({
tech: "React",
tech: "TypeScript",
label: "good-first-issue",
sort: "updated",
linkedPr: "yes",
Expand All @@ -125,6 +125,64 @@ describe("searchGitHubIssues", () => {
expect(result.candidateCount).toBe(1);
});

it("searches framework terms by repository topic instead of issue text", async () => {
const springIssue = githubIssue({
html_url: "https://github.com/spring-projects/spring-boot/issues/123",
repository_url: "https://api.github.com/repos/spring-projects/spring-boot",
title: "Improve actuator diagnostics",
labels: [{ name: "help wanted" }],
});
const fetchMock = vi
.fn()
.mockResolvedValueOnce(
jsonResponse({
total_count: 1,
items: [
{
full_name: "spring-projects/spring-boot",
html_url: "https://github.com/spring-projects/spring-boot",
stargazers_count: 82000,
archived: false,
topics: ["spring-boot"],
},
],
}),
)
.mockResolvedValueOnce(
jsonResponse({
total_count: 1,
items: [springIssue],
}),
);

vi.stubGlobal("fetch", fetchMock);

const result = await searchGitHubIssues({
tech: "Spring Boot",
label: "help-wanted",
sort: "updated",
linkedPr: "any",
});

const repoSearchUrl = new URL(fetchMock.mock.calls[0][0] as string);
const issueSearchUrl = new URL(fetchMock.mock.calls[1][0] as string);
const issueQuery = issueSearchUrl.searchParams.get("q") ?? "";

expect(repoSearchUrl.pathname).toBe("/search/repositories");
expect(repoSearchUrl.searchParams.get("q")).toBe(
"topic:spring-boot archived:false language:Java",
);
expect(issueSearchUrl.pathname).toBe("/search/issues");
expect(issueQuery).toBe(
"is:issue is:open archived:false label:\"help wanted\" repo:spring-projects/spring-boot",
);
expect(issueQuery).not.toContain("Spring Boot");
expect(result.issues[0]).toMatchObject({
repo: "spring-projects/spring-boot",
stars: 82000,
});
});

it("adds the negative linked PR qualifier for no-linked-PR searches", async () => {
const fetchMock = vi.fn();
searchPageResponses([githubIssue()]).forEach((response) => {
Expand Down Expand Up @@ -332,7 +390,7 @@ describe("searchGitHubIssues", () => {
vi.stubGlobal("fetch", fetchMock);

const result = await searchGitHubIssues({
tech: "React",
tech: "TypeScript",
label: "good-first-issue",
sort: "updated",
linkedPr: "any",
Expand Down