From 6731881547563ab187d314c282176e8d83b879c4 Mon Sep 17 00:00:00 2001 From: Spencer Banko Date: Wed, 30 Sep 2026 10:04:03 -0400 Subject: [PATCH 1/5] Add screening pre-check registry lookups MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The lookups half of #30: a shared result type and one entry point, checkRegistries(), that checks volunteers against the Georgia GBI registry (its published CSV) and returns a search-by-hand link for NSOPW, whose terms forbid automated searching. Hits are filtered by birth year (±1). Includes scripts/screening-check.ts, assertion checks that use made-up names only. Co-Authored-By: Claude Opus 5.5 --- package.json | 3 +- scripts/screening-check.ts | 444 +++++++++++++++++++++++++++++ src/constants/limits.ts | 10 + src/lib/csv.ts | 36 +++ src/lib/screening/index.ts | 56 ++++ src/lib/screening/match.ts | 77 +++++ src/lib/screening/source.ts | 35 +++ src/lib/screening/sources/gbi.ts | 164 +++++++++++ src/lib/screening/sources/nsopw.ts | 20 ++ src/types/screening.ts | 59 ++++ 10 files changed, 903 insertions(+), 1 deletion(-) create mode 100644 scripts/screening-check.ts create mode 100644 src/lib/screening/index.ts create mode 100644 src/lib/screening/match.ts create mode 100644 src/lib/screening/source.ts create mode 100644 src/lib/screening/sources/gbi.ts create mode 100644 src/lib/screening/sources/nsopw.ts create mode 100644 src/types/screening.ts diff --git a/package.json b/package.json index 58238d2..a49ab20 100644 --- a/package.json +++ b/package.json @@ -15,7 +15,8 @@ "format:check": "prettier --check \"**/*.{ts,tsx,js,jsx,json,md,mdx,css}\"", "prepare": "husky", "seed": "tsx --env-file=.env.local scripts/seed.ts", - "jobs": "tsx --env-file=.env.local scripts/jobs.ts" + "jobs": "tsx --env-file=.env.local scripts/jobs.ts", + "screening:check": "tsx scripts/screening-check.ts" }, "dependencies": { "@tanstack/react-query": "^5.102.2", diff --git a/scripts/screening-check.ts b/scripts/screening-check.ts new file mode 100644 index 0000000..adc3be0 --- /dev/null +++ b/scripts/screening-check.ts @@ -0,0 +1,444 @@ +/** + * Checks the screening pre-check lookups. There's no test runner yet, so this + * is plain assertions: + * + * npm run screening:check # made-up registry file, no network + * npm run screening:check -- --live # also downloads the real GBI file + * + * Every name here is invented. The live run searches only for made-up names + * and prints outcomes, counts, and timing, never a registry entry. + */ +import assert from "node:assert/strict"; +import { parseCsv, toCsv } from "@/lib/csv"; +import { checkRegistries } from "@/lib/screening"; +import { + birthYearMatches, + namesMatch, + nameWords, + parseRegistryName, +} from "@/lib/screening/match"; +import type { ScreeningSource } from "@/lib/screening/source"; +import { createGbiSource, gbiSource } from "@/lib/screening/sources/gbi"; +import { nsopwSource } from "@/lib/screening/sources/nsopw"; +import type { RegistryCheckResult, RegistrySubject } from "@/types/screening"; + +let passed = 0; +const failures: string[] = []; + +async function test(name: string, run: () => void | Promise) { + try { + await run(); + passed += 1; + console.log(` ok ${name}`); + } catch (error) { + failures.push(name); + console.log(` FAIL ${name}`); + console.log(` ${error instanceof Error ? error.message : error}`); + } +} + +const dob = (value: string) => new Date(`${value}T05:00:00Z`); + +const subject = ( + userId: string, + firstName: string, + lastName: string, + dateOfBirth: string | null +): RegistrySubject => ({ + userId, + firstName, + lastName, + dateOfBirth: dateOfBirth ? dob(dateOfBirth) : null, +}); + +const name = (first: string, last: string) => ({ + first: nameWords(first), + last: nameWords(last), +}); + +/** The GBI header, with a few invented entries. */ +const HEADER = [ + "NAME", + "SEX", + "RACE", + "YEAR OF BIRTH", + "HEIGHT", + "WEIGHT", + "HAIR COLOR", + "EYE COLOR", + "STREET NUMBER", + "STREET", + "CITY", + "STATE", + "ZIP CODE", + "COUNTY", + "REGISTRATION DATE", + "CRIME", +]; +type Entry = { name: string; year: string; city?: string; county?: string }; +const fakeCsv = (entries: Entry[]) => + toCsv( + entries, + HEADER.map((header) => ({ + header, + value: (e: Entry) => + header === "NAME" + ? e.name + : header === "YEAR OF BIRTH" + ? e.year + : header === "CITY" + ? (e.city ?? "FAKETOWN") + : header === "COUNTY" + ? (e.county ?? "NOWHERE") + : "X", + })) + ); + +const FAKE_ENTRIES: Entry[] = [ + { name: "ZZTEST, ALICE MARIE", year: "1990", city: "FAKETOWN" }, + { name: "ZZTEST, ALICE", year: "1991" }, + { name: "ZZTEST, ALICE Q JR", year: "1993" }, + { name: "QXOBRIEN-ZZ, BOBBI", year: "1985" }, + { name: "O'ZZTEST, CARLA", year: "" }, + { name: "ZZTEST, DANA", year: "1970" }, +]; +const fakeGbi = (csv = fakeCsv(FAKE_ENTRIES)) => + createGbiSource({ loadCsv: async () => csv, minEntries: 1 }); + +const only = (results: Map, userId: string) => { + const list = results.get(userId); + assert.ok(list, `no results for ${userId}`); + assert.equal(list.length, 1); + return list[0]; +}; + +async function offline() { + console.log("csv"); + await test("parses quotes, doubled quotes, commas, CRLF, embedded newlines", () => { + const rows = parseCsv( + 'A,B,C\r\n"x, y","say ""hi""","two\nlines"\r\n\r\nplain,,end\n' + ); + assert.deepEqual(rows, [ + ["A", "B", "C"], + ["x, y", 'say "hi"', "two\nlines"], + ["plain", "", "end"], + ]); + }); + await test("round-trips toCsv", () => { + const text = toCsv( + [{ a: 'q"uote', b: "c,omma" }], + [ + { header: "a", value: (r) => r.a }, + { header: "b", value: (r) => r.b }, + ] + ); + assert.deepEqual(parseCsv(text), [ + ["a", "b"], + ['q"uote', "c,omma"], + ]); + }); + + console.log("names"); + await test("normalizes accents, apostrophes, hyphens, suffixes", () => { + assert.deepEqual(nameWords("José"), ["JOSE"]); + assert.deepEqual(nameWords("O'Zztest"), ["OZZTEST"]); + assert.deepEqual(nameWords("Qxobrien-Zz"), ["QXOBRIEN", "ZZ"]); + assert.deepEqual(nameWords("Alice Q Jr."), ["ALICE", "Q"]); + assert.deepEqual(nameWords("V"), ["V"]); + }); + await test("parses LAST, FIRST MIDDLE SUFFIX", () => { + assert.deepEqual(parseRegistryName("ZZTEST, ALICE Q JR"), { + last: ["ZZTEST"], + first: ["ALICE", "Q"], + }); + }); + await test("matches on first + last, ignoring middle names and case", () => { + assert.ok( + namesMatch( + name("alice", "zztest"), + parseRegistryName("ZZTEST, ALICE MARIE") + ) + ); + assert.ok( + namesMatch( + name("Alice Marie", "Zztest"), + parseRegistryName("ZZTEST, ALICE") + ) + ); + }); + await test("does not match a different first or last name", () => { + assert.ok( + !namesMatch(name("Alicia", "Zztest"), parseRegistryName("ZZTEST, ALICE")) + ); + assert.ok( + !namesMatch( + name("Alice", "Zztesting"), + parseRegistryName("ZZTEST, ALICE") + ) + ); + }); + await test("matches compound last names either way", () => { + assert.ok( + namesMatch( + name("Bobbi", "Qxobrien"), + parseRegistryName("QXOBRIEN-ZZ, BOBBI") + ) + ); + assert.ok( + namesMatch( + name("Bobbi", "Qxobrien Zz"), + parseRegistryName("QXOBRIEN-ZZ, BOBBI") + ) + ); + // Short particles alone aren't enough. + assert.ok( + !namesMatch(name("Bobbi", "Zz"), parseRegistryName("QXOBRIEN-ZZ, BOBBI")) + ); + }); + + console.log("birth year"); + await test("same year and ±1 match, 2+ off does not", () => { + assert.ok(birthYearMatches(dob("1990-06-15"), 1990)); + assert.ok(birthYearMatches(dob("1990-06-15"), 1991)); + assert.ok(birthYearMatches(dob("1990-06-15"), 1989)); + assert.ok(!birthYearMatches(dob("1990-06-15"), 1992)); + assert.ok(!birthYearMatches(dob("1990-06-15"), 1987)); + }); + await test("Jan 1 birthdays keep their year (stored as midnight Eastern)", () => { + assert.ok(!birthYearMatches(new Date("1990-01-01T05:00:00Z"), 1988)); + }); + await test("unknown registry year counts as a match", () => { + assert.ok(birthYearMatches(dob("1990-06-15"), null)); + }); + + console.log("GBI source (made-up file)"); + await test("possible match lists every hit within a year, with location and link", async () => { + const results = await checkRegistries( + [subject("u1", "Alice", "Zztest", "1990-03-02")], + [fakeGbi()] + ); + const result = only(results, "u1"); + assert.equal(result.outcome, "possible_match"); + assert.equal(result.source, "gbi_ga"); + assert.ok(result.checkedAt instanceof Date); + if (result.outcome !== "possible_match") return; + // 1990 and 1991 match; 1993 (JR) is out of range. + assert.deepEqual( + result.hits.map((h) => [h.name, h.yearOfBirth]), + [ + ["ZZTEST, ALICE MARIE", 1990], + ["ZZTEST, ALICE", 1991], + ] + ); + assert.equal(result.hits[0].location, "FAKETOWN, NOWHERE County"); + assert.ok(result.hits.every((h) => h.url.startsWith("https://"))); + }); + await test("same name, birth year 3+ off → no match", async () => { + const results = await checkRegistries( + [subject("u1", "Dana", "Zztest", "1990-03-02")], + [fakeGbi()] + ); + assert.equal(only(results, "u1").outcome, "no_match"); + }); + await test("name not in file → no match", async () => { + const results = await checkRegistries( + [subject("u1", "Nobody", "Qqnotreal", "1990-03-02")], + [fakeGbi()] + ); + assert.equal(only(results, "u1").outcome, "no_match"); + }); + await test("blank registry year still flags the name", async () => { + const results = await checkRegistries( + [subject("u1", "Carla", "O'Zztest", "2001-03-02")], + [fakeGbi()] + ); + const result = only(results, "u1"); + assert.equal(result.outcome, "possible_match"); + if (result.outcome === "possible_match") { + assert.equal(result.hits[0].yearOfBirth, null); + } + }); + await test("missing DOB → couldn't check, with manual link, never a name-only flag", async () => { + const results = await checkRegistries( + [subject("u1", "Alice", "Zztest", null)], + [fakeGbi()] + ); + const result = only(results, "u1"); + assert.equal(result.outcome, "unavailable"); + if (result.outcome === "unavailable") { + assert.equal(result.reason, "missing_dob"); + assert.ok(result.manualSearchUrl.startsWith("https://")); + } + }); + await test("no subjects with a DOB → the file isn't downloaded", async () => { + let downloads = 0; + const source = createGbiSource({ + loadCsv: async () => { + downloads += 1; + return fakeCsv(FAKE_ENTRIES); + }, + minEntries: 1, + }); + await checkRegistries([subject("u1", "Alice", "Zztest", null)], [source]); + assert.equal(downloads, 0); + }); + await test("several subjects share one download", async () => { + let downloads = 0; + const source = createGbiSource({ + loadCsv: async () => { + downloads += 1; + return fakeCsv(FAKE_ENTRIES); + }, + minEntries: 1, + }); + const results = await checkRegistries( + [ + subject("u1", "Alice", "Zztest", "1990-03-02"), + subject("u2", "Nobody", "Qqnotreal", "1990-03-02"), + subject("u3", "Bobbi", "Qxobrien", "1986-01-20"), + ], + [source] + ); + assert.equal(downloads, 1); + assert.equal(only(results, "u1").outcome, "possible_match"); + assert.equal(only(results, "u2").outcome, "no_match"); + assert.equal(only(results, "u3").outcome, "possible_match"); + }); + await test("changed columns → couldn't check (source_error), not no match", async () => { + const results = await checkRegistries( + [subject("u1", "Alice", "Zztest", "1990-03-02")], + [fakeGbi("FULL NAME,BORN\nZZTEST ALICE,1990\n")] + ); + const result = only(results, "u1"); + assert.equal(result.outcome, "unavailable"); + if (result.outcome === "unavailable") + assert.equal(result.reason, "source_error"); + }); + await test("truncated file → source_error, not no match", async () => { + const source = createGbiSource({ + loadCsv: async () => fakeCsv(FAKE_ENTRIES), + }); + const results = await checkRegistries( + [subject("u1", "Nobody", "Qqnotreal", "1990-03-02")], + [source] + ); + const result = only(results, "u1"); + assert.equal(result.outcome, "unavailable"); + }); + await test("download failure → source_error", async () => { + const source = createGbiSource({ + loadCsv: async () => { + throw new Error("download failed: 503"); + }, + }); + const results = await checkRegistries( + [subject("u1", "Alice", "Zztest", "1990-03-02")], + [source] + ); + const result = only(results, "u1"); + assert.equal(result.outcome, "unavailable"); + if (result.outcome === "unavailable") + assert.equal(result.reason, "source_error"); + }); + + console.log("NSOPW + checkRegistries"); + await test("NSOPW is always search-by-hand and never fetched", async () => { + const realFetch = globalThis.fetch; + let fetched = false; + globalThis.fetch = (async () => { + fetched = true; + throw new Error("no"); + }) as typeof fetch; + try { + const results = await checkRegistries( + [subject("u1", "Alice", "Zztest", "1990-03-02")], + [nsopwSource] + ); + const result = only(results, "u1"); + assert.equal(result.outcome, "unavailable"); + if (result.outcome === "unavailable") { + assert.equal(result.reason, "manual_only"); + assert.equal(result.manualSearchUrl, "https://www.nsopw.gov/search-0"); + } + assert.equal(fetched, false); + } finally { + globalThis.fetch = realFetch; + } + }); + await test("one result per source per subject, in source order; a broken source doesn't stop the rest", async () => { + const broken: ScreeningSource = { + id: "gbi_ga", + label: "Broken", + manualSearchUrl: "https://example.org", + check: async () => { + throw new Error("boom"); + }, + }; + const results = await checkRegistries( + [ + subject("u1", "Alice", "Zztest", "1990-03-02"), + subject("u2", "Nobody", "Qqnotreal", null), + ], + [broken, nsopwSource] + ); + for (const id of ["u1", "u2"]) { + const list = results.get(id); + assert.deepEqual( + list?.map((r) => [r.source, r.outcome]), + [ + ["gbi_ga", "unavailable"], + ["nsopw", "unavailable"], + ] + ); + } + }); + await test("no subjects → empty map, no source called", async () => { + let called = false; + const spy: ScreeningSource = { + ...nsopwSource, + check: async () => { + called = true; + return new Map(); + }, + }; + const results = await checkRegistries([], [spy]); + assert.equal(results.size, 0); + assert.equal(called, false); + }); +} + +async function live() { + console.log("GBI source (live download, made-up names only)"); + await test("downloads and reads the real file within the job budget", async () => { + const started = Date.now(); + const results = await checkRegistries( + [ + subject("fake-1", "Zzqx", "Notarealperson", "1990-03-02"), + subject("fake-2", "Qqzy", "Madeupname-Testcase", "1975-11-30"), + ], + [gbiSource] + ); + const elapsed = Date.now() - started; + for (const id of ["fake-1", "fake-2"]) { + const result = only(results, id); + assert.equal( + result.outcome, + "no_match", + `expected no_match, got ${result.outcome}${ + result.outcome === "unavailable" ? ` (${result.reason})` : "" + }` + ); + } + console.log(` download + parse + match: ${elapsed} ms`); + assert.ok(elapsed < 8_000, "slower than the fetch timeout"); + }); +} + +async function main() { + await offline(); + if (process.argv.includes("--live")) await live(); + console.log(`\n${passed} passed, ${failures.length} failed`); + if (failures.length) process.exit(1); +} + +main(); diff --git a/src/constants/limits.ts b/src/constants/limits.ts index 452948c..175d282 100644 --- a/src/constants/limits.ts +++ b/src/constants/limits.ts @@ -61,3 +61,13 @@ export const RATE_LIMITS = { verifyEmailResend: { limit: 5, windowMs: 60 * 60_000 }, verifyEmailLookup: { limit: 20, windowMs: 15 * 60_000 }, } as const; + +/** + * Screening pre-check. Registries publish a birth year (or an age), not a + * date, so a name hit counts when its year is within this many years of the + * volunteer's. One year absorbs age rounding and data-entry slips. + */ +export const SCREENING_BIRTH_YEAR_TOLERANCE = 1; + +/** A registry download that answers slower than this is reported as unavailable. */ +export const SCREENING_FETCH_TIMEOUT_MS = 8_000; diff --git a/src/lib/csv.ts b/src/lib/csv.ts index 3c99d65..81e4fb2 100644 --- a/src/lib/csv.ts +++ b/src/lib/csv.ts @@ -17,3 +17,39 @@ export function toCsv(rows: T[], columns: CsvColumn[]): string { } return lines.join("\n"); } + +/** + * Parses RFC 4180 CSV into rows of fields, header included. Handles quoted + * fields, doubled quotes, and line breaks inside quotes; blank lines are skipped. + */ +export function parseCsv(text: string): string[][] { + const rows: string[][] = []; + let row: string[] = []; + let field = ""; + let quoted = false; + + const endRow = () => { + row.push(field); + if (row.length > 1 || row[0] !== "") rows.push(row); + row = []; + field = ""; + }; + + for (let i = 0; i < text.length; i += 1) { + const char = text[i]; + if (quoted) { + if (char !== '"') field += char; + else if (text[i + 1] === '"') { + field += '"'; + i += 1; + } else quoted = false; + } else if (char === '"') quoted = true; + else if (char === ",") { + row.push(field); + field = ""; + } else if (char === "\n") endRow(); + else if (char !== "\r") field += char; + } + if (field !== "" || row.length) endRow(); + return rows; +} diff --git a/src/lib/screening/index.ts b/src/lib/screening/index.ts new file mode 100644 index 0000000..28ef95b --- /dev/null +++ b/src/lib/screening/index.ts @@ -0,0 +1,56 @@ +import { unavailable, type ScreeningSource } from "@/lib/screening/source"; +import { gbiSource } from "@/lib/screening/sources/gbi"; +import { nsopwSource } from "@/lib/screening/sources/nsopw"; +import type { RegistryCheckResult, RegistrySubject } from "@/types/screening"; + +export const SCREENING_SOURCES: ScreeningSource[] = [gbiSource, nsopwSource]; + +/** + * The screening pre-check: every subject against every source, one result + * per source per subject, keyed by `userId`. Never throws. A source that fails + * reports `unavailable`/`source_error` for everyone it was meant to check, + * and the other sources still run. + * + * This only looks things up. It never touches clearance status, and the + * admin decides what a result means. Nothing here logs names or results. + */ +export async function checkRegistries( + subjects: RegistrySubject[], + sources: ScreeningSource[] = SCREENING_SOURCES +): Promise> { + const results = new Map( + subjects.map((s) => [s.userId, []]) + ); + if (!subjects.length) return results; + + const perSource = await Promise.all( + sources.map(async (source) => { + const checkedAt = new Date(); + try { + return { + source, + checkedAt, + found: await source.check(subjects, checkedAt), + }; + } catch (error) { + console.error( + `[screening] ${source.id} failed:`, + error instanceof Error ? error.message : "unknown error" + ); + return { source, checkedAt, found: null }; + } + }) + ); + + for (const { source, checkedAt, found } of perSource) { + for (const subject of subjects) { + results + .get(subject.userId) + ?.push( + found?.get(subject.userId) ?? + unavailable(source, "source_error", checkedAt) + ); + } + } + return results; +} diff --git a/src/lib/screening/match.ts b/src/lib/screening/match.ts new file mode 100644 index 0000000..631247a --- /dev/null +++ b/src/lib/screening/match.ts @@ -0,0 +1,77 @@ +import { SCREENING_BIRTH_YEAR_TOLERANCE } from "@/constants/limits"; + +const SUFFIXES = new Set(["JR", "SR", "II", "III", "IV", "V"]); + +/** Particles like DE or LA are too short to match a compound last name on their own. */ +const MIN_PART_LENGTH = 3; + +/** + * A name as comparable words: accents folded, uppercased, apostrophes and + * periods dropped (O'Brien → OBRIEN), split on anything else (hyphens, + * spaces), and generational suffixes removed unless they're the only word. + */ +export const nameWords = (value: string): string[] => { + const words = value + .normalize("NFD") + .replace(/[̀-ͯ]/g, "") + .toUpperCase() + .replace(/['’.]/g, "") + .split(/[^A-Z]+/) + .filter(Boolean); + const kept = words.filter((word) => !SUFFIXES.has(word)); + return kept.length ? kept : words; +}; + +/** + * The keys a last name can match on: the whole name run together, plus each + * part of a compound name. "Garcia-Lopez" matches a registry "GARCIA" and a + * registry "GARCIA LOPEZ"; the birth year check keeps that from getting noisy. + */ +export const lastNameKeys = (words: string[]): string[] => { + if (!words.length) return []; + const keys = new Set([words.join("")]); + if (words.length > 1) { + for (const word of words) { + if (word.length >= MIN_PART_LENGTH) keys.add(word); + } + } + return [...keys]; +}; + +export interface RegistryName { + first: string[]; + last: string[]; +} + +/** Splits "LAST, FIRST MIDDLE SUFFIX", the format registries print. */ +export const parseRegistryName = (value: string): RegistryName => { + const comma = value.indexOf(","); + if (comma === -1) { + const words = nameWords(value); + return { first: words.slice(0, 1), last: words.slice(-1) }; + } + return { + last: nameWords(value.slice(0, comma)), + first: nameWords(value.slice(comma + 1)), + }; +}; + +/** Last names share a key and first names agree on their first word. Middle names are ignored. */ +export const namesMatch = ( + subject: RegistryName, + entry: RegistryName +): boolean => { + if (!subject.first.length || !entry.first.length) return false; + if (subject.first[0] !== entry.first[0]) return false; + const entryKeys = new Set(lastNameKeys(entry.last)); + return lastNameKeys(subject.last).some((key) => entryKeys.has(key)); +}; + +/** An unknown registry year can't rule a hit out, so it counts. */ +export const birthYearMatches = ( + dateOfBirth: Date, + yearOfBirth: number | null +): boolean => + yearOfBirth == null || + Math.abs(dateOfBirth.getUTCFullYear() - yearOfBirth) <= + SCREENING_BIRTH_YEAR_TOLERANCE; diff --git a/src/lib/screening/source.ts b/src/lib/screening/source.ts new file mode 100644 index 0000000..8250c59 --- /dev/null +++ b/src/lib/screening/source.ts @@ -0,0 +1,35 @@ +import type { + RegistryCheckResult, + RegistrySource, + RegistrySubject, + RegistryUnavailableReason, +} from "@/types/screening"; + +/** + * One place to look someone up. Adding a registry, or swapping them all for a + * vendor API, is a new implementation of this and nothing else. + */ +export interface ScreeningSource { + id: RegistrySource; + label: string; + /** Where an admin searches by hand when the automated check can't answer. */ + manualSearchUrl: string; + /** Checks every subject in one pass. May throw; the caller turns that into `source_error`. */ + check( + subjects: RegistrySubject[], + checkedAt: Date + ): Promise>; +} + +export const unavailable = ( + source: ScreeningSource, + reason: RegistryUnavailableReason, + checkedAt: Date +): RegistryCheckResult => ({ + source: source.id, + sourceLabel: source.label, + checkedAt, + outcome: "unavailable", + reason, + manualSearchUrl: source.manualSearchUrl, +}); diff --git a/src/lib/screening/sources/gbi.ts b/src/lib/screening/sources/gbi.ts new file mode 100644 index 0000000..1783a31 --- /dev/null +++ b/src/lib/screening/sources/gbi.ts @@ -0,0 +1,164 @@ +import { SCREENING_FETCH_TIMEOUT_MS } from "@/constants/limits"; +import { parseCsv } from "@/lib/csv"; +import { + birthYearMatches, + lastNameKeys, + nameWords, + namesMatch, + parseRegistryName, + type RegistryName, +} from "@/lib/screening/match"; +import { unavailable, type ScreeningSource } from "@/lib/screening/source"; +import type { + RegistryCheckResult, + RegistryHit, + RegistrySubject, +} from "@/types/screening"; + +/** + * The Georgia registry, read from the full CSV that GBI publishes for download + * (linked from gbi.georgia.gov as "the latest Georgia sexual offender registry + * data"). One download per run covers every subject, and nothing is scraped. + * + * The file has a birth year but no full date of birth and no per-offender + * link, so hits point at the GBI search page with the entry's city and county. + * Only matching entries survive the call; the file itself is never stored. + */ +const CSV_URL = "https://state.sor.gbi.ga.gov/SORT_PUBLIC/sor.csv"; +const SEARCH_URL = "https://state.sor.gbi.ga.gov/Sort_Public"; + +const COLUMNS = { + name: "NAME", + yearOfBirth: "YEAR OF BIRTH", + city: "CITY", + county: "COUNTY", +} as const; + +/** The real file has ~25,000 entries. Far fewer means a truncated or changed download, not "no match". */ +const MIN_ENTRIES = 1_000; + +const downloadCsv = async (): Promise => { + const response = await fetch(CSV_URL, { + headers: { "User-Agent": "PinkStemVolunteerHub/1.0 (screening pre-check)" }, + signal: AbortSignal.timeout(SCREENING_FETCH_TIMEOUT_MS), + cache: "no-store", + }); + if (!response.ok) throw new Error(`download failed: ${response.status}`); + return response.text(); +}; + +interface Candidate { + subject: RegistrySubject & { dateOfBirth: Date }; + name: RegistryName; +} + +export interface GbiSourceOptions { + /** Replaces the download, for checks against a made-up file. */ + loadCsv?: () => Promise; + minEntries?: number; +} + +export const createGbiSource = ({ + loadCsv = downloadCsv, + minEntries = MIN_ENTRIES, +}: GbiSourceOptions = {}): ScreeningSource => { + const source: ScreeningSource = { + id: "gbi_ga", + label: "Georgia GBI sex offender registry", + manualSearchUrl: SEARCH_URL, + async check(subjects, checkedAt) { + const results = new Map(); + const hits = new Map(); + // Candidates by last-name key, so each registry row is one lookup. + const byKey = new Map(); + + for (const subject of subjects) { + const { dateOfBirth } = subject; + if (!dateOfBirth) { + results.set( + subject.userId, + unavailable(source, "missing_dob", checkedAt) + ); + continue; + } + const candidate: Candidate = { + subject: { ...subject, dateOfBirth }, + name: { + first: nameWords(subject.firstName), + last: nameWords(subject.lastName), + }, + }; + hits.set(subject.userId, []); + for (const key of lastNameKeys(candidate.name.last)) { + byKey.set(key, [...(byKey.get(key) ?? []), candidate]); + } + } + if (!hits.size) return results; + + const [header, ...rows] = parseCsv(await loadCsv()); + const column = Object.fromEntries( + Object.entries(COLUMNS).map(([field, title]) => [ + field, + header?.indexOf(title) ?? -1, + ]) + ) as Record; + if (Object.values(column).some((index) => index === -1)) { + throw new Error("unexpected CSV columns"); + } + if (rows.length < minEntries) { + throw new Error(`only ${rows.length} entries`); + } + + for (const row of rows) { + const printed = row[column.name]?.trim(); + if (!printed) continue; + const entry = parseRegistryName(printed); + const candidates = new Set( + lastNameKeys(entry.last).flatMap((key) => byKey.get(key) ?? []) + ); + if (!candidates.size) continue; + + const year = Number.parseInt(row[column.yearOfBirth] ?? "", 10); + const yearOfBirth = Number.isNaN(year) ? null : year; + for (const { subject, name } of candidates) { + if (!namesMatch(name, entry)) continue; + if (!birthYearMatches(subject.dateOfBirth, yearOfBirth)) continue; + const city = row[column.city]?.trim(); + const county = row[column.county]?.trim(); + hits.get(subject.userId)?.push({ + name: printed, + yearOfBirth, + location: + [city, county && `${county} County`].filter(Boolean).join(", ") || + undefined, + url: SEARCH_URL, + }); + } + } + + for (const [userId, found] of hits) { + results.set( + userId, + found.length + ? { + source: source.id, + sourceLabel: source.label, + checkedAt, + outcome: "possible_match", + hits: found, + } + : { + source: source.id, + sourceLabel: source.label, + checkedAt, + outcome: "no_match", + } + ); + } + return results; + }, + }; + return source; +}; + +export const gbiSource = createGbiSource(); diff --git a/src/lib/screening/sources/nsopw.ts b/src/lib/screening/sources/nsopw.ts new file mode 100644 index 0000000..dca225c --- /dev/null +++ b/src/lib/screening/sources/nsopw.ts @@ -0,0 +1,20 @@ +import { unavailable, type ScreeningSource } from "@/lib/screening/source"; + +/** + * The national registry (NSOPW). Its Conditions of Use forbid "automated + * searching" and its robots.txt disallows /search/, so it is never fetched: + * every subject gets a link for the admin to search by hand. + */ +export const nsopwSource: ScreeningSource = { + id: "nsopw", + label: "National Sex Offender Public Website (NSOPW)", + manualSearchUrl: "https://www.nsopw.gov/search-0", + async check(subjects, checkedAt) { + return new Map( + subjects.map((s) => [ + s.userId, + unavailable(nsopwSource, "manual_only", checkedAt), + ]) + ); + }, +}; diff --git a/src/types/screening.ts b/src/types/screening.ts new file mode 100644 index 0000000..90621f4 --- /dev/null +++ b/src/types/screening.ts @@ -0,0 +1,59 @@ +/** + * The screening pre-check result, shared between the lookups + * (src/lib/screening) and whatever stores and shows them. A vendor API that + * replaces the public registries returns this same shape. + */ +export const REGISTRY_SOURCES = ["gbi_ga", "nsopw"] as const; +export type RegistrySource = (typeof REGISTRY_SOURCES)[number]; + +export const REGISTRY_OUTCOMES = [ + "no_match", + "possible_match", + "unavailable", +] as const; +export type RegistryOutcome = (typeof REGISTRY_OUTCOMES)[number]; + +/** Why a source couldn't be checked automatically. Each comes with a manual link. */ +export const REGISTRY_UNAVAILABLE_REASONS = [ + "manual_only", + "missing_dob", + "source_error", +] as const; +export type RegistryUnavailableReason = + (typeof REGISTRY_UNAVAILABLE_REASONS)[number]; + +/** One registry entry whose name and birth year fit the volunteer. Never the full record. */ +export interface RegistryHit { + /** As the registry prints it, e.g. "DOE, JANE A". */ + name: string; + yearOfBirth: number | null; + /** City and county, so the admin can pick the right entry when verifying. */ + location?: string; + /** Where the admin goes to verify this hit. */ + url: string; +} + +interface RegistryCheckBase { + source: RegistrySource; + sourceLabel: string; + checkedAt: Date; +} + +export type RegistryCheckResult = RegistryCheckBase & + ( + | { outcome: "no_match" } + | { outcome: "possible_match"; hits: RegistryHit[] } + | { + outcome: "unavailable"; + reason: RegistryUnavailableReason; + manualSearchUrl: string; + } + ); + +/** Who to look up. `userId` keys the results; it is never sent to a source. */ +export interface RegistrySubject { + userId: string; + firstName: string; + lastName: string; + dateOfBirth?: Date | null; +} From 08a057d1ec4ed82adc6aab7e8c318cf755087e4c Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 16 Oct 2026 13:57:03 +0000 Subject: [PATCH 2/5] Close the review findings on the registry lookups - Reject a GBI download whose size doesn't match Content-Length, or whose rows don't match the header width, so a cut-off file can't say no match - parseCsv throws on an unclosed quote instead of swallowing the rest - An invalid date of birth reports missing_dob, not no match - A repeated userId gets one result per source - Only ScreeningSourceError messages are logged, so a source can't leak a name into the logs through an exception Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_01GG4LpiuNziAYJVp9WJMbtK --- scripts/screening-check.ts | 82 +++++++++++++++++++++++++++++++- src/lib/csv.ts | 3 ++ src/lib/screening/index.ts | 13 ++++- src/lib/screening/source.ts | 6 +++ src/lib/screening/sources/gbi.ts | 46 +++++++++++++++--- 5 files changed, 140 insertions(+), 10 deletions(-) diff --git a/scripts/screening-check.ts b/scripts/screening-check.ts index adc3be0..f648007 100644 --- a/scripts/screening-check.ts +++ b/scripts/screening-check.ts @@ -17,7 +17,10 @@ import { nameWords, parseRegistryName, } from "@/lib/screening/match"; -import type { ScreeningSource } from "@/lib/screening/source"; +import { + ScreeningSourceError, + type ScreeningSource, +} from "@/lib/screening/source"; import { createGbiSource, gbiSource } from "@/lib/screening/sources/gbi"; import { nsopwSource } from "@/lib/screening/sources/nsopw"; import type { RegistryCheckResult, RegistrySubject } from "@/types/screening"; @@ -124,6 +127,9 @@ async function offline() { ["plain", "", "end"], ]); }); + await test("an unclosed quote throws instead of swallowing the rest", () => { + assert.throws(() => parseCsv('A,B\n"open,1\nnext,2\n')); + }); await test("round-trips toCsv", () => { const text = toCsv( [{ a: 'q"uote', b: "c,omma" }], @@ -325,6 +331,39 @@ async function offline() { const result = only(results, "u1"); assert.equal(result.outcome, "unavailable"); }); + await test("file cut off mid-row → source_error, not no match", async () => { + const csv = fakeCsv(FAKE_ENTRIES); + const results = await checkRegistries( + [subject("u1", "Nobody", "Qqnotreal", "1990-03-02")], + [fakeGbi(csv.slice(0, csv.lastIndexOf(",")))] + ); + const result = only(results, "u1"); + assert.equal(result.outcome, "unavailable"); + if (result.outcome === "unavailable") + assert.equal(result.reason, "source_error"); + }); + await test("unclosed quote late in the file → source_error, not no match", async () => { + const results = await checkRegistries( + [subject("u1", "Nobody", "Qqnotreal", "1990-03-02")], + [fakeGbi(`${fakeCsv(FAKE_ENTRIES)}\n"QQNOTREAL, NOBODY,1990`)] + ); + assert.equal(only(results, "u1").outcome, "unavailable"); + }); + await test("invalid DOB → missing_dob, not no match", async () => { + const results = await checkRegistries( + [ + { + ...subject("u1", "Alice", "Zztest", null), + dateOfBirth: new Date("nope"), + }, + ], + [fakeGbi()] + ); + const result = only(results, "u1"); + assert.equal(result.outcome, "unavailable"); + if (result.outcome === "unavailable") + assert.equal(result.reason, "missing_dob"); + }); await test("download failure → source_error", async () => { const source = createGbiSource({ loadCsv: async () => { @@ -392,6 +431,47 @@ async function offline() { ); } }); + await test("a repeated userId still gets one result per source", async () => { + const results = await checkRegistries( + [ + subject("u1", "Alice", "Zztest", "1990-03-02"), + subject("u1", "Nobody", "Qqnotreal", "1990-03-02"), + ], + [fakeGbi(), nsopwSource] + ); + assert.deepEqual( + results.get("u1")?.map((r) => [r.source, r.outcome]), + [ + ["gbi_ga", "possible_match"], + ["nsopw", "unavailable"], + ] + ); + }); + await test("only ScreeningSourceError messages reach the log", async () => { + const realError = console.error; + const logged: string[] = []; + console.error = (...args: unknown[]) => void logged.push(args.join(" ")); + try { + const throwing = (error: Error): ScreeningSource => ({ + ...nsopwSource, + check: async () => { + throw error; + }, + }); + await checkRegistries( + [subject("u1", "Alice", "Zztest", "1990-03-02")], + [throwing(new ScreeningSourceError("download failed: 503"))] + ); + await checkRegistries( + [subject("u1", "Alice", "Zztest", "1990-03-02")], + [throwing(new Error("no match for Alice Zztest"))] + ); + } finally { + console.error = realError; + } + assert.ok(logged[0]?.includes("download failed: 503")); + assert.ok(!logged[1]?.includes("Zztest")); + }); await test("no subjects → empty map, no source called", async () => { let called = false; const spy: ScreeningSource = { diff --git a/src/lib/csv.ts b/src/lib/csv.ts index 81e4fb2..5b62961 100644 --- a/src/lib/csv.ts +++ b/src/lib/csv.ts @@ -21,6 +21,8 @@ export function toCsv(rows: T[], columns: CsvColumn[]): string { /** * Parses RFC 4180 CSV into rows of fields, header included. Handles quoted * fields, doubled quotes, and line breaks inside quotes; blank lines are skipped. + * Throws on a quote that is never closed, since everything after it would + * otherwise be swallowed into one field. */ export function parseCsv(text: string): string[][] { const rows: string[][] = []; @@ -50,6 +52,7 @@ export function parseCsv(text: string): string[][] { } else if (char === "\n") endRow(); else if (char !== "\r") field += char; } + if (quoted) throw new Error("unclosed quote in CSV"); if (field !== "" || row.length) endRow(); return rows; } diff --git a/src/lib/screening/index.ts b/src/lib/screening/index.ts index 28ef95b..3aa5c1c 100644 --- a/src/lib/screening/index.ts +++ b/src/lib/screening/index.ts @@ -1,4 +1,8 @@ -import { unavailable, type ScreeningSource } from "@/lib/screening/source"; +import { + ScreeningSourceError, + unavailable, + type ScreeningSource, +} from "@/lib/screening/source"; import { gbiSource } from "@/lib/screening/sources/gbi"; import { nsopwSource } from "@/lib/screening/sources/nsopw"; import type { RegistryCheckResult, RegistrySubject } from "@/types/screening"; @@ -18,6 +22,9 @@ export async function checkRegistries( subjects: RegistrySubject[], sources: ScreeningSource[] = SCREENING_SOURCES ): Promise> { + // One subject per userId (the first), so each gets one result per source. + const seen = new Set(); + subjects = subjects.filter((s) => !seen.has(s.userId) && seen.add(s.userId)); const results = new Map( subjects.map((s) => [s.userId, []]) ); @@ -35,7 +42,9 @@ export async function checkRegistries( } catch (error) { console.error( `[screening] ${source.id} failed:`, - error instanceof Error ? error.message : "unknown error" + error instanceof ScreeningSourceError + ? error.message + : "unexpected error" ); return { source, checkedAt, found: null }; } diff --git a/src/lib/screening/source.ts b/src/lib/screening/source.ts index 8250c59..b2e497b 100644 --- a/src/lib/screening/source.ts +++ b/src/lib/screening/source.ts @@ -21,6 +21,12 @@ export interface ScreeningSource { ): Promise>; } +/** + * A failure whose message is safe to log: it names no subject and no result. + * Any other error a source throws is logged without its message. + */ +export class ScreeningSourceError extends Error {} + export const unavailable = ( source: ScreeningSource, reason: RegistryUnavailableReason, diff --git a/src/lib/screening/sources/gbi.ts b/src/lib/screening/sources/gbi.ts index 1783a31..7971b4e 100644 --- a/src/lib/screening/sources/gbi.ts +++ b/src/lib/screening/sources/gbi.ts @@ -8,7 +8,11 @@ import { parseRegistryName, type RegistryName, } from "@/lib/screening/match"; -import { unavailable, type ScreeningSource } from "@/lib/screening/source"; +import { + ScreeningSourceError, + unavailable, + type ScreeningSource, +} from "@/lib/screening/source"; import type { RegistryCheckResult, RegistryHit, @@ -43,8 +47,23 @@ const downloadCsv = async (): Promise => { signal: AbortSignal.timeout(SCREENING_FETCH_TIMEOUT_MS), cache: "no-store", }); - if (!response.ok) throw new Error(`download failed: ${response.status}`); - return response.text(); + if (!response.ok) { + throw new ScreeningSourceError(`download failed: ${response.status}`); + } + const body = await response.arrayBuffer(); + // Content-Length is the encoded size, so it only proves completeness when + // the body wasn't compressed in transit. + const expected = Number(response.headers.get("content-length")); + if ( + !response.headers.get("content-encoding") && + expected && + body.byteLength !== expected + ) { + throw new ScreeningSourceError( + `download incomplete: ${body.byteLength} of ${expected} bytes` + ); + } + return new TextDecoder().decode(body); }; interface Candidate { @@ -74,7 +93,7 @@ export const createGbiSource = ({ for (const subject of subjects) { const { dateOfBirth } = subject; - if (!dateOfBirth) { + if (!dateOfBirth || Number.isNaN(dateOfBirth.getTime())) { results.set( subject.userId, unavailable(source, "missing_dob", checkedAt) @@ -95,7 +114,14 @@ export const createGbiSource = ({ } if (!hits.size) return results; - const [header, ...rows] = parseCsv(await loadCsv()); + const text = await loadCsv(); + let parsed: string[][]; + try { + parsed = parseCsv(text); + } catch { + throw new ScreeningSourceError("unreadable CSV"); + } + const [header, ...rows] = parsed; const column = Object.fromEntries( Object.entries(COLUMNS).map(([field, title]) => [ field, @@ -103,10 +129,16 @@ export const createGbiSource = ({ ]) ) as Record; if (Object.values(column).some((index) => index === -1)) { - throw new Error("unexpected CSV columns"); + throw new ScreeningSourceError("unexpected CSV columns"); } if (rows.length < minEntries) { - throw new Error(`only ${rows.length} entries`); + throw new ScreeningSourceError(`only ${rows.length} entries`); + } + // A download cut off mid-row leaves a short last row; a damaged file + // leaves rows of the wrong width. Either way the file can't vouch for + // "no match". + if (rows.some((row) => row.length !== header.length)) { + throw new ScreeningSourceError("rows don't match the header"); } for (const row of rows) { From 71bfaad4fa949efeab298f2b37fbb02363a8989f Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 16 Oct 2026 14:01:10 +0000 Subject: [PATCH 3/5] Save registry pre-checks and show them to admins - ScreeningCheck collection keeps the latest result per source per volunteer, apart from users like Clearance - The hourly job checks volunteers with a sign-up pending on clearance who were never checked or hit a source error, plus queued reruns - Admins can queue a rerun from the person page; it's audited and runs with the next hourly job - Person page shows each source's result with links to verify hits or search by hand - Approvals adds a Waiting on clearance list next to Clearances to review - Clearance status is never touched Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_01GG4LpiuNziAYJVp9WJMbtK --- src/app/admin/page.tsx | 10 +++ src/app/admin/people/[id]/page.tsx | 39 ++++++++ .../v1/admin/people/[id]/screening/route.ts | 10 +++ src/components/admin/ScreeningResults.tsx | 86 ++++++++++++++++++ src/components/hooks/useAdmin.ts | 4 + src/constants/labels.ts | 27 ++++++ src/db/actions/screeningCheck.ts | 67 ++++++++++++++ src/db/actions/user.ts | 14 +++ src/db/models/screeningCheck.ts | 57 ++++++++++++ src/http/adminHTTPClient.ts | 6 ++ src/services/admin.ts | 75 +++++++++++----- src/services/jobs.ts | 2 + src/services/screening.ts | 88 +++++++++++++++++++ src/types/api.ts | 9 ++ src/types/audit.ts | 1 + src/types/screening.ts | 14 +++ 16 files changed, 485 insertions(+), 24 deletions(-) create mode 100644 src/app/api/v1/admin/people/[id]/screening/route.ts create mode 100644 src/components/admin/ScreeningResults.tsx create mode 100644 src/db/actions/screeningCheck.ts create mode 100644 src/db/models/screeningCheck.ts create mode 100644 src/services/screening.ts diff --git a/src/app/admin/page.tsx b/src/app/admin/page.tsx index 962647e..f42b431 100644 --- a/src/app/admin/page.tsx +++ b/src/app/admin/page.tsx @@ -1,6 +1,7 @@ "use client"; import Link from "next/link"; +import { screeningLine } from "@/components/admin/ScreeningResults"; import { useAdminApprovals } from "@/components/hooks/useAdmin"; import { PageHeader, Spinner } from "@/components/ui/Primitives"; import { formatDate } from "@/lib/dates"; @@ -85,6 +86,15 @@ export default function ApprovalsPage() { empty="No clearances are waiting." rows={a.clearances.map(person)} /> + ({ + ...person(volunteer), + detail: `Waiting on: clearance · ${screeningLine(screening)}`, + }))} + /> + + + run( + () => actions.requestScreeningRerun.mutateAsync(), + "Rerun queued for the next hourly run." + ) + } + > + {p.screening?.rerunRequestedAt ? "Queued" : "Rerun"} + + } + /> + + {p.screening?.rerunRequestedAt ? ( +

+ Rerun queued {formatDateTime(p.screening.rerunRequestedAt)}. + It runs with the next hourly job. +

+ ) : null} + {p.screening?.results.length ? ( + + ) : ( +

+ Not run yet. It runs when this volunteer has a sign-up waiting + on clearance. +

+ )} +
+
diff --git a/src/app/api/v1/admin/people/[id]/screening/route.ts b/src/app/api/v1/admin/people/[id]/screening/route.ts new file mode 100644 index 0000000..e64d749 --- /dev/null +++ b/src/app/api/v1/admin/people/[id]/screening/route.ts @@ -0,0 +1,10 @@ +import { NextResponse } from "next/server"; +import ScreeningService from "@/services/screening"; +import { withAuth } from "@/utils/withAuth"; + +/** Queues a registry pre-check rerun; the hourly job runs it. */ +export const POST = withAuth<{ id: string }>( + async (_req, { params }, actor) => + NextResponse.json(await ScreeningService.requestRerun(actor, params.id)), + { roles: ["admin"] } +); diff --git a/src/components/admin/ScreeningResults.tsx b/src/components/admin/ScreeningResults.tsx new file mode 100644 index 0000000..38855d0 --- /dev/null +++ b/src/components/admin/ScreeningResults.tsx @@ -0,0 +1,86 @@ +import { ReactNode } from "react"; +import Badge from "@/components/ui/Badge"; +import { + REGISTRY_OUTCOME_LABELS, + REGISTRY_SOURCE_SHORT_LABELS, + REGISTRY_UNAVAILABLE_LABELS, +} from "@/constants/labels"; +import type { ClientScreeningCheck } from "@/http/adminHTTPClient"; +import { formatDateTime } from "@/lib/dates"; +import type { RegistryCheckResult } from "@/types/screening"; +import type { Serialized } from "@/types/models"; + +type Result = Serialized; + +const TONE = { + no_match: "neutral", + possible_match: "warning", + unavailable: "info", +} as const; + +const outcomeLabel = (r: Result) => + r.outcome === "unavailable" + ? `${REGISTRY_OUTCOME_LABELS.unavailable}: ${REGISTRY_UNAVAILABLE_LABELS[r.reason].toLowerCase()}` + : REGISTRY_OUTCOME_LABELS[r.outcome]; + +/** One line for the Approvals queue, e.g. "GBI: No match · NSOPW: Search by hand". */ +export function screeningLine(check: ClientScreeningCheck | null): string { + if (check?.rerunRequestedAt) return "Pre-check rerun queued"; + if (!check?.results.length) return "Pre-check not run yet"; + return check.results + .map( + (r) => + `${REGISTRY_SOURCE_SHORT_LABELS[r.source]}: ${ + r.outcome === "unavailable" + ? REGISTRY_UNAVAILABLE_LABELS[r.reason] + : REGISTRY_OUTCOME_LABELS[r.outcome] + }` + ) + .join(" · "); +} + +const link = (href: string, children: ReactNode) => ( + + {children} + +); + +/** Every source's saved result, with links to verify each hit or search by hand. */ +export default function ScreeningResults({ results }: { results: Result[] }) { + return ( +
    + {results.map((r) => ( +
  • +
    + {r.sourceLabel} + {outcomeLabel(r)} +
    +
    + Checked {formatDateTime(r.checkedAt)} +
    + {r.outcome === "possible_match" ? ( +
      + {r.hits.map((hit, i) => ( +
    • + {link(hit.url, hit.name)} + {hit.yearOfBirth ? ` · born ${hit.yearOfBirth}` : ""} + {hit.location ? ` · ${hit.location}` : ""} +
    • + ))} +
    + ) : null} + {r.outcome === "unavailable" ? ( +
    + {link(r.manualSearchUrl, "Search this registry by hand")} +
    + ) : null} +
  • + ))} +
+ ); +} diff --git a/src/components/hooks/useAdmin.ts b/src/components/hooks/useAdmin.ts index 38e3ad8..1570e86 100644 --- a/src/components/hooks/useAdmin.ts +++ b/src/components/hooks/useAdmin.ts @@ -112,6 +112,10 @@ export function useAdminActions(userId?: string) { AdminHTTPClient.recordClearance(id(), body), onSuccess: invalidate, }), + requestScreeningRerun: useMutation({ + mutationFn: () => AdminHTTPClient.requestScreeningRerun(id()), + onSuccess: invalidate, + }), forceSignout: useMutation({ mutationFn: () => AdminHTTPClient.forceSignout(id()), onSuccess: invalidate, diff --git a/src/constants/labels.ts b/src/constants/labels.ts index e621b00..02717f2 100644 --- a/src/constants/labels.ts +++ b/src/constants/labels.ts @@ -1,6 +1,11 @@ import type { AuditAction } from "@/types/audit"; import type { CertificateType } from "@/types/certificate"; import type { EventStatus, ProgramArea, UpdateKind } from "@/types/event"; +import type { + RegistryOutcome, + RegistrySource, + RegistryUnavailableReason, +} from "@/types/screening"; import type { PendingReason, SignupStatus } from "@/types/signup"; import type { ClearanceStatus, @@ -116,6 +121,7 @@ export const AUDIT_ACTION_LABELS: Record = { "user.force_signout": "Forced sign-out", "user.flagged_for_review": "Flagged for review", "clearance.recorded": "Clearance recorded", + "screening.rerun_requested": "Registry pre-check rerun requested", "hours.approved": "Hours approved", "hours.adjusted": "Hours adjusted", "certificate.issued": "Certificate issued", @@ -130,3 +136,24 @@ export const AUDIT_ACTION_LABELS: Record = { "shift.counters_reconciled": "Shift counters reconciled", "auth.rate_limited": "Rate limit triggered", }; + +/** Short source names for tight spaces like the Approvals queue. */ +export const REGISTRY_SOURCE_SHORT_LABELS: Record = { + gbi_ga: "GBI", + nsopw: "NSOPW", +}; + +export const REGISTRY_OUTCOME_LABELS: Record = { + no_match: "No match", + possible_match: "Possible match", + unavailable: "Couldn't check", +}; + +export const REGISTRY_UNAVAILABLE_LABELS: Record< + RegistryUnavailableReason, + string +> = { + manual_only: "Search by hand", + missing_dob: "No date of birth", + source_error: "Source error, retrying hourly", +}; diff --git a/src/db/actions/screeningCheck.ts b/src/db/actions/screeningCheck.ts new file mode 100644 index 0000000..00537c6 --- /dev/null +++ b/src/db/actions/screeningCheck.ts @@ -0,0 +1,67 @@ +import { Types } from "mongoose"; +import dbConnect from "@/db/dbConnect"; +import ScreeningCheckModel from "@/db/models/screeningCheck"; +import type { Doc } from "@/types/models"; +import type { RegistryCheckResult, ScreeningCheck } from "@/types/screening"; + +export default class ScreeningCheckDAO { + static async findByUser( + userId: string | Types.ObjectId + ): Promise | null> { + await dbConnect(); + return ScreeningCheckModel.findOne({ userId }).lean>(); + } + + static async findByUsers( + userIds: (string | Types.ObjectId)[] + ): Promise[]> { + await dbConnect(); + return ScreeningCheckModel.find({ userId: { $in: userIds } }).lean< + Doc[] + >(); + } + + static async findRerunRequested(): Promise[]> { + await dbConnect(); + return ScreeningCheckModel.find({ rerunRequestedAt: { $ne: null } }).lean< + Doc[] + >(); + } + + static async requestRerun( + userId: string | Types.ObjectId, + adminId: string | Types.ObjectId, + at: Date + ): Promise> { + await dbConnect(); + return ScreeningCheckModel.findOneAndUpdate( + { userId }, + { + rerunRequestedAt: at, + rerunRequestedBy: new Types.ObjectId(adminId), + }, + { upsert: true, returnDocument: "after", runValidators: true } + ).lean>() as Promise>; + } + + /** + * Replaces the saved results. A rerun request made before `startedAt` is + * answered by this run and cleared; one made during the run stays queued. + */ + static async saveResults( + userId: string | Types.ObjectId, + results: RegistryCheckResult[], + startedAt: Date + ): Promise { + await dbConnect(); + await ScreeningCheckModel.updateOne( + { userId }, + { results }, + { upsert: true, runValidators: true } + ); + await ScreeningCheckModel.updateOne( + { userId, rerunRequestedAt: { $lte: startedAt } }, + { rerunRequestedAt: null, rerunRequestedBy: null } + ); + } +} diff --git a/src/db/actions/user.ts b/src/db/actions/user.ts index 8be4311..0f84e1c 100644 --- a/src/db/actions/user.ts +++ b/src/db/actions/user.ts @@ -118,6 +118,20 @@ export default class UserDAO { .lean(); } + /** Just what the screening pre-check needs to look someone up. */ + static async findScreeningSubjects( + ids: (string | Types.ObjectId)[] + ): Promise< + Pick, "_id" | "firstName" | "lastName" | "dateOfBirth">[] + > { + await dbConnect(); + return UserModel.find({ _id: { $in: ids } }) + .select("firstName lastName dateOfBirth") + .lean< + Pick, "_id" | "firstName" | "lastName" | "dateOfBirth">[] + >(); + } + static async listByRole(role: Role): Promise[]> { await dbConnect(); return UserModel.find({ role, status: "active" }).lean[]>(); diff --git a/src/db/models/screeningCheck.ts b/src/db/models/screeningCheck.ts new file mode 100644 index 0000000..9862c2e --- /dev/null +++ b/src/db/models/screeningCheck.ts @@ -0,0 +1,57 @@ +import { Schema } from "mongoose"; +import { defineModel } from "@/db/defineModel"; +import { + REGISTRY_OUTCOMES, + REGISTRY_SOURCES, + REGISTRY_UNAVAILABLE_REASONS, + ScreeningCheck, +} from "@/types/screening"; + +const hitSchema = new Schema( + { + name: { type: String, required: true }, + yearOfBirth: { type: Number, default: null }, + location: String, + url: { type: String, required: true }, + }, + { _id: false } +); + +/** One RegistryCheckResult, stored as-is. Fields outside its outcome stay unset. */ +const resultSchema = new Schema( + { + source: { type: String, enum: REGISTRY_SOURCES, required: true }, + sourceLabel: { type: String, required: true }, + checkedAt: { type: Date, required: true }, + outcome: { type: String, enum: REGISTRY_OUTCOMES, required: true }, + hits: { type: [hitSchema], default: undefined }, + reason: { type: String, enum: REGISTRY_UNAVAILABLE_REASONS }, + manualSearchUrl: String, + }, + { _id: false } +); + +/** + * The registry pre-check for a volunteer. Kept apart from users, like + * Clearance, so it is never returned alongside a user by accident. It never + * changes clearance; the admin decides what a result means. + */ +const screeningCheckSchema = new Schema( + { + userId: { + type: Schema.ObjectId, + ref: "User", + required: true, + unique: true, + }, + results: { type: [resultSchema], default: [] }, + rerunRequestedAt: { type: Date, default: null }, + rerunRequestedBy: { type: Schema.ObjectId, ref: "User", default: null }, + }, + { timestamps: true } +); + +export default defineModel( + "ScreeningCheck", + screeningCheckSchema +); diff --git a/src/http/adminHTTPClient.ts b/src/http/adminHTTPClient.ts index 5f46ca7..c4d0de2 100644 --- a/src/http/adminHTTPClient.ts +++ b/src/http/adminHTTPClient.ts @@ -10,6 +10,7 @@ import type { Report, } from "@/types/api"; import type { Doc, Serialized } from "@/types/models"; +import type { ScreeningCheck } from "@/types/screening"; import type { OrgSettings } from "@/types/settings"; import type { Clearance, Role } from "@/types/user"; import type { @@ -27,6 +28,7 @@ export type ClientPersonDetail = Serialized; export type ClientAuditRow = Serialized; export type ClientReport = Serialized; export type ClientClearance = Serialized>; +export type ClientScreeningCheck = Serialized>; export type ClearanceBody = Omit & { clearedOn?: string | null; expiresOn?: string | null; @@ -76,6 +78,10 @@ export default class AdminHTTPClient { return fetchHTTPClient(`/admin/people/${userId}/clearance`, "PUT", body); } + static requestScreeningRerun(userId: string): Promise { + return fetchHTTPClient(`/admin/people/${userId}/screening`, "POST"); + } + static forceSignout(userId: string): Promise { return fetchHTTPClient(`/admin/people/${userId}/signout`, "POST"); } diff --git a/src/services/admin.ts b/src/services/admin.ts index c95e23a..030a595 100644 --- a/src/services/admin.ts +++ b/src/services/admin.ts @@ -3,6 +3,7 @@ import ActionTokenDAO from "@/db/actions/actionToken"; import AuditLogDAO from "@/db/actions/auditLog"; import CertificateDAO from "@/db/actions/certificate"; import ClearanceDAO from "@/db/actions/clearance"; +import ScreeningCheckDAO from "@/db/actions/screeningCheck"; import EventDAO from "@/db/actions/event"; import HoursLedgerDAO from "@/db/actions/hoursLedger"; import MessageThreadDAO from "@/db/actions/messageThread"; @@ -54,21 +55,38 @@ const RESERVED_ACTOR_NAMES: Record = { export default class AdminService { /** What is waiting on a decision: one list per kind of approval. */ static async approvals(admin: Actor): Promise { - const [submitted, rosters, consentSignups, reported] = await Promise.all([ - ClearanceDAO.findUserIdsByStatus("submitted"), - EventDAO.findAll({ status: "published", eventDate: { $lt: new Date() } }), - SignupDAO.find({ status: "pending", pendingReasons: "guardian_consent" }), - MessageService.reportedThreads(admin), - ]); - const [clearances, guardianConsent] = await Promise.all([ - UserDAO.findSummaries(submitted), - UserDAO.findSummaries( - consentSignups.map((s) => s.volunteerId), - { guardianConsentAt: null } - ), - ]); + const [submitted, rosters, consentSignups, clearanceSignups, reported] = + await Promise.all([ + ClearanceDAO.findUserIdsByStatus("submitted"), + EventDAO.findAll({ + status: "published", + eventDate: { $lt: new Date() }, + }), + SignupDAO.find({ + status: "pending", + pendingReasons: "guardian_consent", + }), + SignupDAO.find({ status: "pending", pendingReasons: "clearance" }), + MessageService.reportedThreads(admin), + ]); + const awaitingIds = clearanceSignups.map((s) => s.volunteerId); + const [clearances, guardianConsent, awaiting, screenings] = + await Promise.all([ + UserDAO.findSummaries(submitted), + UserDAO.findSummaries( + consentSignups.map((s) => s.volunteerId), + { guardianConsentAt: null } + ), + UserDAO.findSummaries(awaitingIds), + ScreeningCheckDAO.findByUsers(awaitingIds), + ]); return { clearances, + awaitingClearance: awaiting.map((volunteer) => ({ + volunteer, + screening: + screenings.find((c) => sameId(c.userId, volunteer._id)) ?? null, + })), rosters, guardianConsent, reportedThreads: reported, @@ -115,20 +133,29 @@ export default class AdminService { static async getPerson(userId: string): Promise { const user = await UserDAO.findById(userId); if (!user) throw new NotFoundError(ERRORS.USER.NOT_FOUND); - const [clearance, signups, ledger, certificates, audit, threadCount] = - await Promise.all([ - ClearanceDAO.findByUser(user._id), - SignupDAO.findByVolunteer(user._id), - HoursLedgerDAO.findByVolunteer(user._id), - CertificateDAO.findByVolunteer(user._id), - AuditLogDAO.findForUser(user._id), - MessageThreadDAO.count({ - $or: [{ volunteerId: user._id }, { organizerId: user._id }], - }), - ]); + const [ + clearance, + screening, + signups, + ledger, + certificates, + audit, + threadCount, + ] = await Promise.all([ + ClearanceDAO.findByUser(user._id), + ScreeningCheckDAO.findByUser(user._id), + SignupDAO.findByVolunteer(user._id), + HoursLedgerDAO.findByVolunteer(user._id), + CertificateDAO.findByVolunteer(user._id), + AuditLogDAO.findForUser(user._id), + MessageThreadDAO.count({ + $or: [{ volunteerId: user._id }, { organizerId: user._id }], + }), + ]); return { user, clearance, + screening, signups: await SignupService.withContext(signups), ledger: await HoursService.withEvents(ledger), certificates, diff --git a/src/services/jobs.ts b/src/services/jobs.ts index 58a7e70..a5d6c2c 100644 --- a/src/services/jobs.ts +++ b/src/services/jobs.ts @@ -19,6 +19,7 @@ import ClearanceService from "@/services/clearance"; import EventUpdateService from "@/services/eventUpdate"; import MessageService from "@/services/message"; import NotificationService from "@/services/notification"; +import ScreeningService from "@/services/screening"; import SignupService from "@/services/signup"; import { sameId } from "@/utils/authorization"; @@ -56,6 +57,7 @@ export default class JobService { rosterNudges: JobService.rosterNudges, clearanceWarnings: ClearanceService.warnExpiring, clearanceExpiry: ClearanceService.expireLapsed, + screening: ScreeningService.runDue, holdExpiry: SignupService.expireHolds, noteDigest: EventUpdateService.sendNoteDigest, messageDigest: MessageService.sendDigests, diff --git a/src/services/screening.ts b/src/services/screening.ts new file mode 100644 index 0000000..479fba2 --- /dev/null +++ b/src/services/screening.ts @@ -0,0 +1,88 @@ +import ScreeningCheckDAO from "@/db/actions/screeningCheck"; +import SignupDAO from "@/db/actions/signup"; +import UserDAO from "@/db/actions/user"; +import { checkRegistries } from "@/lib/screening"; +import AuditService from "@/services/audit"; +import type { Actor } from "@/types/auth"; +import { NotFoundError } from "@/types/exceptions"; +import type { Doc } from "@/types/models"; +import type { ScreeningCheck } from "@/types/screening"; +import ERRORS from "@/utils/errorMessages"; + +/** + * The registry pre-check, saved for admins. It runs lookups only from the + * hourly job, never inside a request, and it never changes clearance: the + * admin reads the result and still decides. Nothing here logs names or results. + */ +export default class ScreeningService { + /** + * Scheduled: checks every volunteer who needs it and saves the results. + * That is anyone with a sign-up pending on clearance who has never been + * checked or whose last check hit a source error, plus anyone an admin + * queued for a rerun. Returns how many volunteers were checked. + */ + static async runDue(): Promise { + const startedAt = new Date(); + const [pending, queued] = await Promise.all([ + SignupDAO.find({ status: "pending", pendingReasons: "clearance" }), + ScreeningCheckDAO.findRerunRequested(), + ]); + const pendingIds = [ + ...new Set(pending.map((s) => s.volunteerId.toString())), + ]; + const saved = new Map( + (await ScreeningCheckDAO.findByUsers(pendingIds)).map((c) => [ + c.userId.toString(), + c, + ]) + ); + const due = new Set([ + ...pendingIds.filter((id) => { + const check = saved.get(id); + return ( + !check?.results.length || + check.results.some( + (r) => r.outcome === "unavailable" && r.reason === "source_error" + ) + ); + }), + ...queued.map((c) => c.userId.toString()), + ]); + if (!due.size) return 0; + + const users = await UserDAO.findScreeningSubjects([...due]); + const results = await checkRegistries( + users.map((u) => ({ + userId: u._id.toString(), + firstName: u.firstName, + lastName: u.lastName, + dateOfBirth: u.dateOfBirth ?? null, + })) + ); + for (const [userId, found] of results) { + await ScreeningCheckDAO.saveResults(userId, found, startedAt); + } + return results.size; + } + + /** Queues a fresh check for the next hourly run. */ + static async requestRerun( + admin: Actor, + userId: string + ): Promise> { + const user = await UserDAO.findById(userId); + if (!user) throw new NotFoundError(ERRORS.USER.NOT_FOUND); + const check = await ScreeningCheckDAO.requestRerun( + user._id, + admin.id, + new Date() + ); + await AuditService.record( + admin, + "screening.rerun_requested", + "user", + user._id + ); + return check; + } +} diff --git a/src/types/api.ts b/src/types/api.ts index e3d9b01..108e6cc 100644 --- a/src/types/api.ts +++ b/src/types/api.ts @@ -10,6 +10,7 @@ import type { UserSummary, } from "@/types/user"; import type { AuditLog } from "@/types/audit"; +import type { ScreeningCheck } from "@/types/screening"; /** * Composite shapes the API returns. Client code reads them through @@ -109,6 +110,7 @@ export type PersonRow = Doc & { export interface PersonDetail { user: Doc; clearance: Doc | null; + screening: Doc | null; signups: SignupWithContext[]; ledger: LedgerEntryWithEvent[]; certificates: Doc[]; @@ -132,8 +134,15 @@ export interface Report { summary: { label: string; value: string }[]; } +/** A volunteer with a sign-up pending on clearance, and their pre-check. */ +export interface AwaitingClearance { + volunteer: UserSummary; + screening: Doc | null; +} + export interface AdminApprovals { clearances: UserSummary[]; + awaitingClearance: AwaitingClearance[]; rosters: Doc[]; guardianConsent: UserSummary[]; reportedThreads: ThreadSummary[]; diff --git a/src/types/audit.ts b/src/types/audit.ts index aac5d63..a3e67d0 100644 --- a/src/types/audit.ts +++ b/src/types/audit.ts @@ -16,6 +16,7 @@ export const AUDIT_ACTIONS = [ "user.force_signout", "user.flagged_for_review", "clearance.recorded", + "screening.rerun_requested", "hours.approved", "hours.adjusted", "certificate.issued", diff --git a/src/types/screening.ts b/src/types/screening.ts index 90621f4..2b8ac3b 100644 --- a/src/types/screening.ts +++ b/src/types/screening.ts @@ -1,3 +1,5 @@ +import type { Types } from "mongoose"; + /** * The screening pre-check result, shared between the lookups * (src/lib/screening) and whatever stores and shows them. A vendor API that @@ -57,3 +59,15 @@ export interface RegistrySubject { lastName: string; dateOfBirth?: Date | null; } + +/** + * The saved pre-check for one volunteer: the latest result from each source. + * Admin-only, like the clearance record, and kept in its own collection. + */ +export interface ScreeningCheck { + userId: Types.ObjectId; + results: RegistryCheckResult[]; + /** Set when an admin asks for a rerun; the next hourly job clears it. */ + rerunRequestedAt: Date | null; + rerunRequestedBy: Types.ObjectId | null; +} From 5fb5241b4f94ada22d2271514147dbaf03181312 Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 16 Oct 2026 14:03:39 +0000 Subject: [PATCH 4/5] Recheck volunteers who add a date of birth after a missing_dob result Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_01GG4LpiuNziAYJVp9WJMbtK --- src/constants/labels.ts | 2 +- src/services/screening.ts | 45 ++++++++++++++++++++++++--------------- 2 files changed, 29 insertions(+), 18 deletions(-) diff --git a/src/constants/labels.ts b/src/constants/labels.ts index 02717f2..f16648d 100644 --- a/src/constants/labels.ts +++ b/src/constants/labels.ts @@ -154,6 +154,6 @@ export const REGISTRY_UNAVAILABLE_LABELS: Record< string > = { manual_only: "Search by hand", - missing_dob: "No date of birth", + missing_dob: "No date of birth; rechecks once one is added", source_error: "Source error, retrying hourly", }; diff --git a/src/services/screening.ts b/src/services/screening.ts index 479fba2..996fd5a 100644 --- a/src/services/screening.ts +++ b/src/services/screening.ts @@ -6,7 +6,10 @@ import AuditService from "@/services/audit"; import type { Actor } from "@/types/auth"; import { NotFoundError } from "@/types/exceptions"; import type { Doc } from "@/types/models"; -import type { ScreeningCheck } from "@/types/screening"; +import type { + RegistryUnavailableReason, + ScreeningCheck, +} from "@/types/screening"; import ERRORS from "@/utils/errorMessages"; /** @@ -18,8 +21,9 @@ export default class ScreeningService { /** * Scheduled: checks every volunteer who needs it and saves the results. * That is anyone with a sign-up pending on clearance who has never been - * checked or whose last check hit a source error, plus anyone an admin - * queued for a rerun. Returns how many volunteers were checked. + * checked, whose last check hit a source error, or who had no date of birth + * then and has one now, plus anyone an admin queued for a rerun. Returns + * how many volunteers were checked. */ static async runDue(): Promise { const startedAt = new Date(); @@ -30,29 +34,36 @@ export default class ScreeningService { const pendingIds = [ ...new Set(pending.map((s) => s.volunteerId.toString())), ]; + const queuedIds = new Set(queued.map((c) => c.userId.toString())); const saved = new Map( (await ScreeningCheckDAO.findByUsers(pendingIds)).map((c) => [ c.userId.toString(), c, ]) ); - const due = new Set([ - ...pendingIds.filter((id) => { - const check = saved.get(id); - return ( - !check?.results.length || - check.results.some( - (r) => r.outcome === "unavailable" && r.reason === "source_error" - ) - ); - }), - ...queued.map((c) => c.userId.toString()), + const users = await UserDAO.findScreeningSubjects([ + ...new Set([...pendingIds, ...queuedIds]), ]); - if (!due.size) return 0; - const users = await UserDAO.findScreeningSubjects([...due]); + const failed = (id: string, reason: RegistryUnavailableReason) => + saved + .get(id) + ?.results.some( + (r) => r.outcome === "unavailable" && r.reason === reason + ); + const due = users.filter((u) => { + const id = u._id.toString(); + if (queuedIds.has(id)) return true; + if (!saved.get(id)?.results.length) return true; + if (failed(id, "source_error")) return true; + const hasDob = + !!u.dateOfBirth && !Number.isNaN(new Date(u.dateOfBirth).getTime()); + return hasDob && failed(id, "missing_dob"); + }); + if (!due.length) return 0; + const results = await checkRegistries( - users.map((u) => ({ + due.map((u) => ({ userId: u._id.toString(), firstName: u.firstName, lastName: u.lastName, From eb9b1558ffac683af3049cd25e0a5870aad4ff33 Mon Sep 17 00:00:00 2001 From: sailalithkanumuri8 Date: Mon, 19 Oct 2026 13:09:13 -0400 Subject: [PATCH 5/5] Close the gaps found reviewing the registry pre-check MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Fold Latin letters that NFD leaves alone (Đ, Ł, Ø, Æ, Œ, Þ), and report a name with letters the matcher still can't read as "couldn't check" (unmatchable_name) instead of a false "no match" - Recheck a volunteer when a pending sign-up or their profile changes after the last check, so a lapsed clearance or a corrected name or date of birth gets a fresh look; the missing_dob case is covered by the same rule - Treat a quote inside an unquoted CSV value as text, so an inch mark cannot swallow the rows after it - Count a registry birth year that isn't a plain year as unknown, raise the entry floor to 20,000 so a cut-off compressed download can't pass, and log fetch failures by their error type - Run the pre-check last in the hourly job, and save volunteers in parallel - Drop rerunRequestedBy (the audit row records who asked), show a Verify link per hit instead of linking the name, and document the collection, route, script and job --- README.md | 25 +++---- docs/ARCHITECTURE.md | 6 +- netlify/functions/scheduled-jobs.mts | 3 +- scripts/screening-check.ts | 86 ++++++++++++++++++++++- src/app/admin/page.tsx | 2 +- src/components/admin/ScreeningResults.tsx | 25 ++++--- src/constants/labels.ts | 1 + src/db/actions/screeningCheck.ts | 8 +-- src/db/actions/user.ts | 17 ++--- src/db/models/screeningCheck.ts | 1 - src/lib/csv.ts | 7 +- src/lib/screening/index.ts | 2 +- src/lib/screening/match.ts | 39 +++++++--- src/lib/screening/sources/gbi.ts | 71 ++++++++++++------- src/services/admin.ts | 2 +- src/services/jobs.ts | 3 +- src/services/screening.ts | 75 ++++++++++---------- src/types/screening.ts | 4 +- 18 files changed, 250 insertions(+), 127 deletions(-) diff --git a/README.md b/README.md index 9bbcd76..be3bb73 100644 --- a/README.md +++ b/README.md @@ -82,17 +82,18 @@ in VSCode. A pre-commit hook formats and lints staged files automatically. ## Scripts -| Command | Does | -| ---------------------- | ------------------------------------------- | -| `npm run dev` | Start the dev server | -| `npm run build` | Production build | -| `npm run lint` | ESLint | -| `npm run typecheck` | `tsc --noEmit` | -| `npm run format` | Prettier write | -| `npm run format:check` | Prettier check (runs in CI) | -| `npm run seed` | Create settings and the first admin account | -| `npm run seed:staging` | Seed the staging database with demo data | -| `npm run jobs` | Run the scheduled jobs against a dev server | +| Command | Does | +| ------------------------- | ------------------------------------------- | +| `npm run dev` | Start the dev server | +| `npm run build` | Production build | +| `npm run lint` | ESLint | +| `npm run typecheck` | `tsc --noEmit` | +| `npm run format` | Prettier write | +| `npm run format:check` | Prettier check (runs in CI) | +| `npm run seed` | Create settings and the first admin account | +| `npm run seed:staging` | Seed the staging database with demo data | +| `npm run jobs` | Run the scheduled jobs against a dev server | +| `npm run screening:check` | Offline checks for the registry pre-check | ## API @@ -108,7 +109,7 @@ authorization is enforced server-side on every request by `withAuth`. | Sign-ups | `signups`, `signups/:id/{cancel,approve,promote,attendance,calendar}` | | Messaging | `threads`, `threads/:id`, `threads/:id/messages`, `threads/:id/report`, `threads/:id/review` | | Certificates | `certificates/:id/pdf`, `certificates/:id/revoke` | -| Admin | `admin/{approvals,people,people/:id,people/:id/clearance,people/:id/signout,invitations,organizers,events,hours,audit,reports/:kind,settings}` | +| Admin | `admin/{approvals,people,people/:id,people/:id/{clearance,screening,signout},invitations,organizers,events,hours,audit,reports/:kind,settings}` | | Jobs | `jobs/run` (bearer `CRON_SECRET`) | ## Structure diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md index 6206430..57cfa23 100644 --- a/docs/ARCHITECTURE.md +++ b/docs/ARCHITECTURE.md @@ -26,7 +26,7 @@ src/app/**/page.tsx UI. No fetch calls, no business rules. | `src/db/actions/` | DAO classes with static methods | | `src/db/models/` | Mongoose schemas; the interfaces they implement live in `types` | | `src/http/` | Typed frontend API clients | -| `src/lib/` | Session signing, dates, tokens, email templates, PDF, CSV, ICS | +| `src/lib/` | Session, dates, tokens, emails, PDF, CSV, ICS, registry lookups | | `src/middleware/` | Response header helpers | | `src/services/` | Business logic classes with static methods | | `src/styles/` | Global CSS and font configuration | @@ -77,6 +77,7 @@ through a hook in `src/components/hooks/`, so no screen fetches from | ------------------ | ------------------------------------------------------------------------------- | | `users` | Accounts, profile, waiver acceptance, notification preferences, session version | | `clearances` | Screening outcome per user; admins only. Organizers receive a derived boolean | +| `screeningchecks` | Latest registry pre-check per volunteer and any queued rerun; admins only | | `events` | Organizer-owned events: draft → published → completed, cancelled from any state | | `shifts` | The unit volunteers sign up for; `filledCount` is maintained atomically | | `signups` | One per volunteer per shift, ever; pending reasons; embedded attendance | @@ -104,7 +105,8 @@ Two decisions carry most of the weight: `JobService.runAll` (`POST /api/v1/jobs/run`, through `runExclusive`) sends shift reminders, low-fill alerts, unapproved-roster nudges, clearance expiry warnings, note and message -digests, and the organizer digest, and closes stale threads. Every send is +digests, and the organizer digest, closes stale threads, and runs the registry +pre-check for volunteers waiting on clearance. Every send is keyed in `notificationlogs`, so the runner is safe to fire hourly and a late or repeated run is harmless. `JobService.runExclusive` holds an expiring lock in the same collection, so a manual run never overlaps the scheduled one. diff --git a/netlify/functions/scheduled-jobs.mts b/netlify/functions/scheduled-jobs.mts index 0d7e497..4be8ac0 100644 --- a/netlify/functions/scheduled-jobs.mts +++ b/netlify/functions/scheduled-jobs.mts @@ -1,7 +1,8 @@ import type { Config } from "@netlify/functions"; /** - * Hourly trigger for reminders, digests, clearance expiry, and roster nudges. + * Hourly trigger for reminders, digests, clearance expiry, roster nudges, and + * the registry pre-check. * The work itself lives in the app (POST /api/v1/jobs/run) so it shares the * services and the database connection; this function only rings the bell. */ diff --git a/scripts/screening-check.ts b/scripts/screening-check.ts index f648007..ea43de1 100644 --- a/scripts/screening-check.ts +++ b/scripts/screening-check.ts @@ -13,6 +13,7 @@ import { parseCsv, toCsv } from "@/lib/csv"; import { checkRegistries } from "@/lib/screening"; import { birthYearMatches, + isMatchable, namesMatch, nameWords, parseRegistryName, @@ -104,6 +105,8 @@ const FAKE_ENTRIES: Entry[] = [ { name: "QXOBRIEN-ZZ, BOBBI", year: "1985" }, { name: "O'ZZTEST, CARLA", year: "" }, { name: "ZZTEST, DANA", year: "1970" }, + { name: "ZZLOWSKI, MICHAL", year: "1990" }, + { name: "ZZDATE, ERIN", year: "03/02/1990" }, ]; const fakeGbi = (csv = fakeCsv(FAKE_ENTRIES)) => createGbiSource({ loadCsv: async () => csv, minEntries: 1 }); @@ -130,6 +133,13 @@ async function offline() { await test("an unclosed quote throws instead of swallowing the rest", () => { assert.throws(() => parseCsv('A,B\n"open,1\nnext,2\n')); }); + await test("a quote inside a value is text, not the start of a field", () => { + assert.deepEqual(parseCsv(`A,B\n5'10",x\nnext,y\n`), [ + ["A", "B"], + [`5'10"`, "x"], + ["next", "y"], + ]); + }); await test("round-trips toCsv", () => { const text = toCsv( [{ a: 'q"uote', b: "c,omma" }], @@ -152,6 +162,17 @@ async function offline() { assert.deepEqual(nameWords("Alice Q Jr."), ["ALICE", "Q"]); assert.deepEqual(nameWords("V"), ["V"]); }); + await test("folds Latin letters that have no accent to strip", () => { + assert.deepEqual(nameWords("Đặng"), ["DANG"]); + assert.deepEqual(nameWords("Michał"), ["MICHAL"]); + assert.deepEqual(nameWords("Søren Æsa"), ["SOREN", "AESA"]); + assert.deepEqual(nameWords("Þóra Straße"), ["THORA", "STRASSE"]); + }); + await test("a name in another script can't be matched", () => { + assert.ok(isMatchable("Zoë O'Zztest-Đặng")); + assert.ok(!isMatchable("李")); + assert.ok(!isMatchable("Иван")); + }); await test("parses LAST, FIRST MIDDLE SUFFIX", () => { assert.deepEqual(parseRegistryName("ZZTEST, ALICE Q JR"), { last: ["ZZTEST"], @@ -216,6 +237,16 @@ async function offline() { await test("unknown registry year counts as a match", () => { assert.ok(birthYearMatches(dob("1990-06-15"), null)); }); + await test("a registry year that isn't a plain year counts as unknown", async () => { + const results = await checkRegistries( + [subject("u1", "Erin", "Zzdate", "1970-01-01")], + [fakeGbi()] + ); + const result = only(results, "u1"); + assert.equal(result.outcome, "possible_match"); + if (result.outcome === "possible_match") + assert.equal(result.hits[0].yearOfBirth, null); + }); console.log("GBI source (made-up file)"); await test("possible match lists every hit within a year, with location and link", async () => { @@ -264,6 +295,23 @@ async function offline() { assert.equal(result.hits[0].yearOfBirth, null); } }); + await test("an accented name matches its plain registry spelling", async () => { + const results = await checkRegistries( + [subject("u1", "Michał", "Zzłowski", "1990-03-02")], + [fakeGbi()] + ); + assert.equal(only(results, "u1").outcome, "possible_match"); + }); + await test("a name that can't be matched → couldn't check, never no match", async () => { + const results = await checkRegistries( + [subject("u1", "李", "Zztest", "1990-03-02")], + [fakeGbi()] + ); + const result = only(results, "u1"); + assert.equal(result.outcome, "unavailable"); + if (result.outcome === "unavailable") + assert.equal(result.reason, "unmatchable_name"); + }); await test("missing DOB → couldn't check, with manual link, never a name-only flag", async () => { const results = await checkRegistries( [subject("u1", "Alice", "Zztest", null)], @@ -330,6 +378,8 @@ async function offline() { ); const result = only(results, "u1"); assert.equal(result.outcome, "unavailable"); + if (result.outcome === "unavailable") + assert.equal(result.reason, "source_error"); }); await test("file cut off mid-row → source_error, not no match", async () => { const csv = fakeCsv(FAKE_ENTRIES); @@ -347,7 +397,10 @@ async function offline() { [subject("u1", "Nobody", "Qqnotreal", "1990-03-02")], [fakeGbi(`${fakeCsv(FAKE_ENTRIES)}\n"QQNOTREAL, NOBODY,1990`)] ); - assert.equal(only(results, "u1").outcome, "unavailable"); + const result = only(results, "u1"); + assert.equal(result.outcome, "unavailable"); + if (result.outcome === "unavailable") + assert.equal(result.reason, "source_error"); }); await test("invalid DOB → missing_dob, not no match", async () => { const results = await checkRegistries( @@ -469,8 +522,35 @@ async function offline() { } finally { console.error = realError; } - assert.ok(logged[0]?.includes("download failed: 503")); - assert.ok(!logged[1]?.includes("Zztest")); + assert.equal(logged.length, 2); + assert.ok(logged[0].includes("download failed: 503")); + assert.ok(logged[1].includes("unexpected error")); + assert.ok(!logged[1].includes("Zztest")); + }); + await test("a failed download is logged by its error type, never its details", async () => { + const realFetch = globalThis.fetch; + const realError = console.error; + const logged: string[] = []; + globalThis.fetch = (async () => { + throw new TypeError("fetch failed for Alice Zztest"); + }) as typeof fetch; + console.error = (...args: unknown[]) => void logged.push(args.join(" ")); + try { + const results = await checkRegistries( + [subject("u1", "Alice", "Zztest", "1990-03-02")], + [gbiSource] + ); + const result = only(results, "u1"); + assert.equal(result.outcome, "unavailable"); + if (result.outcome === "unavailable") + assert.equal(result.reason, "source_error"); + } finally { + globalThis.fetch = realFetch; + console.error = realError; + } + assert.deepEqual(logged, [ + "[screening] gbi_ga failed: download failed: TypeError", + ]); }); await test("no subjects → empty map, no source called", async () => { let called = false; diff --git a/src/app/admin/page.tsx b/src/app/admin/page.tsx index f42b431..af8cf23 100644 --- a/src/app/admin/page.tsx +++ b/src/app/admin/page.tsx @@ -92,7 +92,7 @@ export default function ApprovalsPage() { empty="No sign-ups are waiting on clearance." rows={a.awaitingClearance.map(({ volunteer, screening }) => ({ ...person(volunteer), - detail: `Waiting on: clearance · ${screeningLine(screening)}`, + detail: `${volunteer.email} · ${screeningLine(screening)}`, }))} /> +/** The outcome, or for "couldn't check" the reason why. */ +const resultLabel = (r: Result) => r.outcome === "unavailable" - ? `${REGISTRY_OUTCOME_LABELS.unavailable}: ${REGISTRY_UNAVAILABLE_LABELS[r.reason].toLowerCase()}` + ? REGISTRY_UNAVAILABLE_LABELS[r.reason] : REGISTRY_OUTCOME_LABELS[r.outcome]; /** One line for the Approvals queue, e.g. "GBI: No match · NSOPW: Search by hand". */ @@ -28,14 +29,7 @@ export function screeningLine(check: ClientScreeningCheck | null): string { if (check?.rerunRequestedAt) return "Pre-check rerun queued"; if (!check?.results.length) return "Pre-check not run yet"; return check.results - .map( - (r) => - `${REGISTRY_SOURCE_SHORT_LABELS[r.source]}: ${ - r.outcome === "unavailable" - ? REGISTRY_UNAVAILABLE_LABELS[r.reason] - : REGISTRY_OUTCOME_LABELS[r.outcome] - }` - ) + .map((r) => `${REGISTRY_SOURCE_SHORT_LABELS[r.source]}: ${resultLabel(r)}`) .join(" · "); } @@ -58,7 +52,11 @@ export default function ScreeningResults({ results }: { results: Result[] }) {
  • {r.sourceLabel} - {outcomeLabel(r)} + + {r.outcome === "unavailable" + ? `${REGISTRY_OUTCOME_LABELS.unavailable}: ${resultLabel(r).toLowerCase()}` + : resultLabel(r)} +
    Checked {formatDateTime(r.checkedAt)} @@ -67,9 +65,10 @@ export default function ScreeningResults({ results }: { results: Result[] }) {
      {r.hits.map((hit, i) => (
    • - {link(hit.url, hit.name)} + {hit.name} {hit.yearOfBirth ? ` · born ${hit.yearOfBirth}` : ""} - {hit.location ? ` · ${hit.location}` : ""} + {hit.location ? ` · ${hit.location}` : ""} ·{" "} + {link(hit.url, "Verify")}
    • ))}
    diff --git a/src/constants/labels.ts b/src/constants/labels.ts index f16648d..25c280a 100644 --- a/src/constants/labels.ts +++ b/src/constants/labels.ts @@ -155,5 +155,6 @@ export const REGISTRY_UNAVAILABLE_LABELS: Record< > = { manual_only: "Search by hand", missing_dob: "No date of birth; rechecks once one is added", + unmatchable_name: "Name can't be matched automatically", source_error: "Source error, retrying hourly", }; diff --git a/src/db/actions/screeningCheck.ts b/src/db/actions/screeningCheck.ts index 00537c6..5557baa 100644 --- a/src/db/actions/screeningCheck.ts +++ b/src/db/actions/screeningCheck.ts @@ -30,16 +30,12 @@ export default class ScreeningCheckDAO { static async requestRerun( userId: string | Types.ObjectId, - adminId: string | Types.ObjectId, at: Date ): Promise> { await dbConnect(); return ScreeningCheckModel.findOneAndUpdate( { userId }, - { - rerunRequestedAt: at, - rerunRequestedBy: new Types.ObjectId(adminId), - }, + { rerunRequestedAt: at }, { upsert: true, returnDocument: "after", runValidators: true } ).lean>() as Promise>; } @@ -61,7 +57,7 @@ export default class ScreeningCheckDAO { ); await ScreeningCheckModel.updateOne( { userId, rerunRequestedAt: { $lte: startedAt } }, - { rerunRequestedAt: null, rerunRequestedBy: null } + { rerunRequestedAt: null } ); } } diff --git a/src/db/actions/user.ts b/src/db/actions/user.ts index 0f84e1c..d943ccb 100644 --- a/src/db/actions/user.ts +++ b/src/db/actions/user.ts @@ -29,6 +29,11 @@ export interface UserSearch { page: number; } +type ScreeningSubject = Pick< + Doc, + "_id" | "firstName" | "lastName" | "dateOfBirth" | "updatedAt" +>; + const SUMMARY_FIELDS = "firstName lastName email phone skills role status"; /** @@ -118,18 +123,14 @@ export default class UserDAO { .lean(); } - /** Just what the screening pre-check needs to look someone up. */ + /** Just what the screening pre-check needs, plus when it last changed. */ static async findScreeningSubjects( ids: (string | Types.ObjectId)[] - ): Promise< - Pick, "_id" | "firstName" | "lastName" | "dateOfBirth">[] - > { + ): Promise { await dbConnect(); return UserModel.find({ _id: { $in: ids } }) - .select("firstName lastName dateOfBirth") - .lean< - Pick, "_id" | "firstName" | "lastName" | "dateOfBirth">[] - >(); + .select("firstName lastName dateOfBirth updatedAt") + .lean(); } static async listByRole(role: Role): Promise[]> { diff --git a/src/db/models/screeningCheck.ts b/src/db/models/screeningCheck.ts index 9862c2e..9d657b7 100644 --- a/src/db/models/screeningCheck.ts +++ b/src/db/models/screeningCheck.ts @@ -46,7 +46,6 @@ const screeningCheckSchema = new Schema( }, results: { type: [resultSchema], default: [] }, rerunRequestedAt: { type: Date, default: null }, - rerunRequestedBy: { type: Schema.ObjectId, ref: "User", default: null }, }, { timestamps: true } ); diff --git a/src/lib/csv.ts b/src/lib/csv.ts index 5b62961..c0cff47 100644 --- a/src/lib/csv.ts +++ b/src/lib/csv.ts @@ -21,8 +21,9 @@ export function toCsv(rows: T[], columns: CsvColumn[]): string { /** * Parses RFC 4180 CSV into rows of fields, header included. Handles quoted * fields, doubled quotes, and line breaks inside quotes; blank lines are skipped. - * Throws on a quote that is never closed, since everything after it would - * otherwise be swallowed into one field. + * A quote only opens a field at its start, so one inside a value (5'10") is + * kept as text. Throws on a quote that is never closed, since everything + * after it would otherwise be swallowed into one field. */ export function parseCsv(text: string): string[][] { const rows: string[][] = []; @@ -45,7 +46,7 @@ export function parseCsv(text: string): string[][] { field += '"'; i += 1; } else quoted = false; - } else if (char === '"') quoted = true; + } else if (char === '"' && field === "") quoted = true; else if (char === ",") { row.push(field); field = ""; diff --git a/src/lib/screening/index.ts b/src/lib/screening/index.ts index 3aa5c1c..851e137 100644 --- a/src/lib/screening/index.ts +++ b/src/lib/screening/index.ts @@ -7,7 +7,7 @@ import { gbiSource } from "@/lib/screening/sources/gbi"; import { nsopwSource } from "@/lib/screening/sources/nsopw"; import type { RegistryCheckResult, RegistrySubject } from "@/types/screening"; -export const SCREENING_SOURCES: ScreeningSource[] = [gbiSource, nsopwSource]; +const SCREENING_SOURCES: ScreeningSource[] = [gbiSource, nsopwSource]; /** * The screening pre-check: every subject against every source, one result diff --git a/src/lib/screening/match.ts b/src/lib/screening/match.ts index 631247a..5e2661b 100644 --- a/src/lib/screening/match.ts +++ b/src/lib/screening/match.ts @@ -5,17 +5,40 @@ const SUFFIXES = new Set(["JR", "SR", "II", "III", "IV", "V"]); /** Particles like DE or LA are too short to match a compound last name on their own. */ const MIN_PART_LENGTH = 3; +/** Latin letters that NFD does not split into a base letter and an accent. */ +const LETTERS: Record = { + Æ: "AE", + Ð: "D", + Đ: "D", + Ł: "L", + Ø: "O", + Œ: "OE", + Þ: "TH", +}; + +/** Uppercased A–Z where possible: accents folded, apostrophes and periods dropped. */ +const fold = (value: string) => + value + .normalize("NFD") + .replace(/[\u0300-\u036f]/g, "") + .toUpperCase() + .replace(/[ÆÐĐŁØŒÞ]/g, (letter) => LETTERS[letter]) + .replace(/['’.]/g, ""); + /** - * A name as comparable words: accents folded, uppercased, apostrophes and - * periods dropped (O'Brien → OBRIEN), split on anything else (hyphens, - * spaces), and generational suffixes removed unless they're the only word. + * Whether a name folds entirely to A–Z. Letters that do not (another script, + * say) would be dropped, so a "no match" for that name would mean nothing. + */ +export const isMatchable = (value: string) => + !/[\u0080-\uffff]/.test(fold(value)); + +/** + * A name as comparable words: folded (O'Brien → OBRIEN), split on anything + * else (hyphens, spaces), and generational suffixes removed unless they're + * the only word. */ export const nameWords = (value: string): string[] => { - const words = value - .normalize("NFD") - .replace(/[̀-ͯ]/g, "") - .toUpperCase() - .replace(/['’.]/g, "") + const words = fold(value) .split(/[^A-Z]+/) .filter(Boolean); const kept = words.filter((word) => !SUFFIXES.has(word)); diff --git a/src/lib/screening/sources/gbi.ts b/src/lib/screening/sources/gbi.ts index 7971b4e..0191c04 100644 --- a/src/lib/screening/sources/gbi.ts +++ b/src/lib/screening/sources/gbi.ts @@ -2,6 +2,7 @@ import { SCREENING_FETCH_TIMEOUT_MS } from "@/constants/limits"; import { parseCsv } from "@/lib/csv"; import { birthYearMatches, + isMatchable, lastNameKeys, nameWords, namesMatch, @@ -38,19 +39,34 @@ const COLUMNS = { county: "COUNTY", } as const; -/** The real file has ~25,000 entries. Far fewer means a truncated or changed download, not "no match". */ -const MIN_ENTRIES = 1_000; +/** + * The real file has ~25,600 entries. Fewer than this means a truncated or + * changed download, which can't vouch for "no match". A compressed download + * cut off on a row boundary passes every other check, so the floor is close. + */ +const MIN_ENTRIES = 20_000; const downloadCsv = async (): Promise => { - const response = await fetch(CSV_URL, { - headers: { "User-Agent": "PinkStemVolunteerHub/1.0 (screening pre-check)" }, - signal: AbortSignal.timeout(SCREENING_FETCH_TIMEOUT_MS), - cache: "no-store", - }); + let response: Response; + let body: ArrayBuffer; + try { + response = await fetch(CSV_URL, { + headers: { + "User-Agent": "PinkStemVolunteerHub/1.0 (screening pre-check)", + }, + signal: AbortSignal.timeout(SCREENING_FETCH_TIMEOUT_MS), + cache: "no-store", + }); + body = await response.arrayBuffer(); + } catch (error) { + // A timeout or network error names no subject, so its type is safe to log. + throw new ScreeningSourceError( + `download failed: ${error instanceof Error ? error.name : "unknown"}` + ); + } if (!response.ok) { throw new ScreeningSourceError(`download failed: ${response.status}`); } - const body = await response.arrayBuffer(); // Content-Length is the encoded size, so it only proves completeness when // the body wasn't compressed in transit. const expected = Number(response.headers.get("content-length")); @@ -71,7 +87,7 @@ interface Candidate { name: RegistryName; } -export interface GbiSourceOptions { +interface GbiSourceOptions { /** Replaces the download, for checks against a made-up file. */ loadCsv?: () => Promise; minEntries?: number; @@ -107,6 +123,19 @@ export const createGbiSource = ({ last: nameWords(subject.lastName), }, }; + // A name the matcher can't read would always come back "no match". + if ( + !isMatchable(subject.firstName) || + !isMatchable(subject.lastName) || + !candidate.name.first.length || + !lastNameKeys(candidate.name.last).length + ) { + results.set( + subject.userId, + unavailable(source, "unmatchable_name", checkedAt) + ); + continue; + } hits.set(subject.userId, []); for (const key of lastNameKeys(candidate.name.last)) { byKey.set(key, [...(byKey.get(key) ?? []), candidate]); @@ -121,7 +150,8 @@ export const createGbiSource = ({ } catch { throw new ScreeningSourceError("unreadable CSV"); } - const [header, ...rows] = parsed; + const [rawHeader, ...rows] = parsed; + const header = rawHeader?.map((title) => title.trim()); const column = Object.fromEntries( Object.entries(COLUMNS).map(([field, title]) => [ field, @@ -150,8 +180,9 @@ export const createGbiSource = ({ ); if (!candidates.size) continue; - const year = Number.parseInt(row[column.yearOfBirth] ?? "", 10); - const yearOfBirth = Number.isNaN(year) ? null : year; + // Anything but a plain year is unknown, which can't rule a hit out. + const year = row[column.yearOfBirth]?.trim() ?? ""; + const yearOfBirth = /^\d{4}$/.test(year) ? Number(year) : null; for (const { subject, name } of candidates) { if (!namesMatch(name, entry)) continue; if (!birthYearMatches(subject.dateOfBirth, yearOfBirth)) continue; @@ -168,23 +199,13 @@ export const createGbiSource = ({ } } + const base = { source: source.id, sourceLabel: source.label, checkedAt }; for (const [userId, found] of hits) { results.set( userId, found.length - ? { - source: source.id, - sourceLabel: source.label, - checkedAt, - outcome: "possible_match", - hits: found, - } - : { - source: source.id, - sourceLabel: source.label, - checkedAt, - outcome: "no_match", - } + ? { ...base, outcome: "possible_match", hits: found } + : { ...base, outcome: "no_match" } ); } return results; diff --git a/src/services/admin.ts b/src/services/admin.ts index 030a595..33c316e 100644 --- a/src/services/admin.ts +++ b/src/services/admin.ts @@ -3,10 +3,10 @@ import ActionTokenDAO from "@/db/actions/actionToken"; import AuditLogDAO from "@/db/actions/auditLog"; import CertificateDAO from "@/db/actions/certificate"; import ClearanceDAO from "@/db/actions/clearance"; -import ScreeningCheckDAO from "@/db/actions/screeningCheck"; import EventDAO from "@/db/actions/event"; import HoursLedgerDAO from "@/db/actions/hoursLedger"; import MessageThreadDAO from "@/db/actions/messageThread"; +import ScreeningCheckDAO from "@/db/actions/screeningCheck"; import SignupDAO from "@/db/actions/signup"; import UserDAO from "@/db/actions/user"; import { ORGANIZER_INVITE_DAYS, PAGE_SIZE } from "@/constants/limits"; diff --git a/src/services/jobs.ts b/src/services/jobs.ts index a5d6c2c..e8275ad 100644 --- a/src/services/jobs.ts +++ b/src/services/jobs.ts @@ -57,12 +57,13 @@ export default class JobService { rosterNudges: JobService.rosterNudges, clearanceWarnings: ClearanceService.warnExpiring, clearanceExpiry: ClearanceService.expireLapsed, - screening: ScreeningService.runDue, holdExpiry: SignupService.expireHolds, noteDigest: EventUpdateService.sendNoteDigest, messageDigest: MessageService.sendDigests, staleThreads: MessageService.closeStale, organizerDigest: JobService.organizerDigest, + // Last, so a slow registry download never holds up the emails. + screening: ScreeningService.runDue, }; const results: Record = {}; for (const [name, job] of Object.entries(jobs)) { diff --git a/src/services/screening.ts b/src/services/screening.ts index 996fd5a..06f43e1 100644 --- a/src/services/screening.ts +++ b/src/services/screening.ts @@ -6,10 +6,7 @@ import AuditService from "@/services/audit"; import type { Actor } from "@/types/auth"; import { NotFoundError } from "@/types/exceptions"; import type { Doc } from "@/types/models"; -import type { - RegistryUnavailableReason, - ScreeningCheck, -} from "@/types/screening"; +import type { ScreeningCheck } from "@/types/screening"; import ERRORS from "@/utils/errorMessages"; /** @@ -20,10 +17,10 @@ import ERRORS from "@/utils/errorMessages"; export default class ScreeningService { /** * Scheduled: checks every volunteer who needs it and saves the results. - * That is anyone with a sign-up pending on clearance who has never been - * checked, whose last check hit a source error, or who had no date of birth - * then and has one now, plus anyone an admin queued for a rerun. Returns - * how many volunteers were checked. + * That is anyone with a sign-up pending on clearance who has not been + * checked since that sign-up or their profile last changed, or whose last + * check hit a source error, plus anyone an admin queued for a rerun. + * Returns how many volunteers were checked. */ static async runDue(): Promise { const startedAt = new Date(); @@ -31,34 +28,36 @@ export default class ScreeningService { SignupDAO.find({ status: "pending", pendingReasons: "clearance" }), ScreeningCheckDAO.findRerunRequested(), ]); - const pendingIds = [ - ...new Set(pending.map((s) => s.volunteerId.toString())), - ]; + // When each volunteer's pending sign-ups last changed. + const signupsChanged = new Map(); + for (const signup of pending) { + const id = signup.volunteerId.toString(); + signupsChanged.set( + id, + Math.max(signupsChanged.get(id) ?? 0, +signup.updatedAt) + ); + } const queuedIds = new Set(queued.map((c) => c.userId.toString())); - const saved = new Map( - (await ScreeningCheckDAO.findByUsers(pendingIds)).map((c) => [ - c.userId.toString(), - c, - ]) - ); - const users = await UserDAO.findScreeningSubjects([ - ...new Set([...pendingIds, ...queuedIds]), + const ids = [...new Set([...signupsChanged.keys(), ...queuedIds])]; + const [users, checks] = await Promise.all([ + UserDAO.findScreeningSubjects(ids), + ScreeningCheckDAO.findByUsers(ids), ]); + const saved = new Map(checks.map((c) => [c.userId.toString(), c.results])); - const failed = (id: string, reason: RegistryUnavailableReason) => - saved - .get(id) - ?.results.some( - (r) => r.outcome === "unavailable" && r.reason === reason - ); const due = users.filter((u) => { const id = u._id.toString(); - if (queuedIds.has(id)) return true; - if (!saved.get(id)?.results.length) return true; - if (failed(id, "source_error")) return true; - const hasDob = - !!u.dateOfBirth && !Number.isNaN(new Date(u.dateOfBirth).getTime()); - return hasDob && failed(id, "missing_dob"); + const results = saved.get(id) ?? []; + if (queuedIds.has(id) || !results.length) return true; + if ( + results.some( + (r) => r.outcome === "unavailable" && r.reason === "source_error" + ) + ) + return true; + // A new sign-up, or a changed name or date of birth, needs a fresh look. + const checkedAt = Math.min(...results.map((r) => +r.checkedAt)); + return Math.max(+u.updatedAt, signupsChanged.get(id) ?? 0) > checkedAt; }); if (!due.length) return 0; @@ -70,9 +69,11 @@ export default class ScreeningService { dateOfBirth: u.dateOfBirth ?? null, })) ); - for (const [userId, found] of results) { - await ScreeningCheckDAO.saveResults(userId, found, startedAt); - } + await Promise.all( + [...results].map(([userId, found]) => + ScreeningCheckDAO.saveResults(userId, found, startedAt) + ) + ); return results.size; } @@ -83,11 +84,7 @@ export default class ScreeningService { ): Promise> { const user = await UserDAO.findById(userId); if (!user) throw new NotFoundError(ERRORS.USER.NOT_FOUND); - const check = await ScreeningCheckDAO.requestRerun( - user._id, - admin.id, - new Date() - ); + const check = await ScreeningCheckDAO.requestRerun(user._id, new Date()); await AuditService.record( admin, "screening.rerun_requested", diff --git a/src/types/screening.ts b/src/types/screening.ts index 2b8ac3b..a81c957 100644 --- a/src/types/screening.ts +++ b/src/types/screening.ts @@ -19,6 +19,7 @@ export type RegistryOutcome = (typeof REGISTRY_OUTCOMES)[number]; export const REGISTRY_UNAVAILABLE_REASONS = [ "manual_only", "missing_dob", + "unmatchable_name", "source_error", ] as const; export type RegistryUnavailableReason = @@ -52,7 +53,7 @@ export type RegistryCheckResult = RegistryCheckBase & } ); -/** Who to look up. `userId` keys the results; it is never sent to a source. */ +/** Who to look up. `userId` keys the results; it never leaves the app. */ export interface RegistrySubject { userId: string; firstName: string; @@ -69,5 +70,4 @@ export interface ScreeningCheck { results: RegistryCheckResult[]; /** Set when an admin asks for a rerun; the next hourly job clears it. */ rerunRequestedAt: Date | null; - rerunRequestedBy: Types.ObjectId | null; }