A house crawler gets its own rung on the two profile routes (#195)

nichedb.dev pulls every author's OpenProfile.md into its people
collection and names itself in its user agent. On the free rung (120 a
minute, 600 an hour) a backfill of two hundred thousand authors takes
days. A request that names a house crawler AND asks for
/api/openprofiles or an author's openprofile.md is placed on a `house`
rung: 600 a minute, 36,000 an hour. Every other route and every other
caller is unchanged, and a session or a key still places the caller by
what it carries.


Claude-Session: https://claude.ai/code/session_01S7yeJUHGxA4P5N74xnsRPQ

Co-authored-by: Claude Fable 5.1 <noreply@anthropic.com>

Author: ralyodio

apps/web/src/lib/tiers.js

@@ -117,8 +117,45 @@ export const TIERS = {
    * unchanged here.
    */
   pass: { name: 'pass', burst: envInt('TIER_SPONSOR_BURST', 2_000), hourly: SPONSOR_HOURLY },
+  /**
+   * A house crawler on the profile routes.
+   *
+   * nichedb.dev pulls every author's OpenProfile.md into its people collection
+   * (nichedb.dev/c/profiles). It names itself in its user agent and wants two
+   * things only: the listing at /api/openprofiles and the files the listing
+   * names. Under the free rung (120 a minute, 600 an hour) a backfill of two
+   * hundred thousand authors takes days; at 600 a minute it takes hours. The
+   * rung exists for those two routes and no other, so a stranger wearing the
+   * string gains a signed-in reader's pace on the two cheapest routes on the
+   * site and nothing anywhere else.
+   */
+  house: { name: 'house', burst: envInt('TIER_HOUSE_BURST', 600), hourly: envInt('TIER_HOUSE_HOURLY', 36_000) },
 };
 
+/** User agent prefixes of the house crawlers, as they identify themselves. */
+export const HOUSE_CRAWLERS = ['niche-db/'];
+
+/** The routes the house rung applies to, and nothing else. */
+const HOUSE_ROUTES = [/^\/api\/openprofiles$/, /^\/authors\/[^/]+\/openprofile\.md$/];
+
+/**
+ * Whether this request is a house crawler asking for a profile route.
+ *
+ * @param {Request} request
+ * @returns {boolean}
+ */
+export function isHouseCrawl(request) {
+  const ua = (request.headers.get('user-agent') ?? '').trim();
+  if (!HOUSE_CRAWLERS.some((prefix) => ua.startsWith(prefix))) return false;
+  let pathname;
+  try {
+    pathname = /** @type {any} */ (request).nextUrl?.pathname ?? new URL(request.url).pathname;
+  } catch {
+    return false;
+  }
+  return HOUSE_ROUTES.some((re) => re.test(pathname));
+}
+
 /**
  * Validated sponsor keys, by hash, with the time they were checked.
  *
@@ -204,6 +241,11 @@ export function tierFor(request) {
 
   if (/(^|;\s*)rsa_session=[^;]/.test(request.headers.get('cookie') ?? '')) return TIERS.session;
 
+  // Below the cookie and the key on purpose: a house crawler carrying either
+  // is placed by what it carries, and the string only matters when it is all
+  // the request has to say for itself.
+  if (isHouseCrawl(request)) return TIERS.house;
+
   return TIERS.anon;
 }
 

apps/web/test/tiers.test.js

@@ -219,3 +219,25 @@ test('the tiers are separately metered, so one does not spend another', () => {
 function hashOf(token) {
   return hashToken(token);
 }
+
+test('a house crawler gets the house rung on the two profile routes and nowhere else', () => {
+  resetTierCache();
+  const ua = 'niche-db/0.1 (+https://nichedb.dev)';
+  const at = (path, agent = ua) =>
+    tierFor(new Request(`https://rssamplifier.com${path}`, { headers: agent ? { 'user-agent': agent } : {} })).name;
+  assert.equal(at('/api/openprofiles?limit=500'), 'house');
+  assert.equal(at('/authors/ada-lovelace/openprofile.md'), 'house');
+  // Same crawler, any other route: free, as before.
+  assert.equal(at('/authors/ada-lovelace'), 'anon');
+  assert.equal(at('/api/authors'), 'anon');
+  assert.equal(at('/topics'), 'anon');
+  // Same routes, any other caller: free, as before.
+  assert.equal(at('/api/openprofiles', 'curl/8.0'), 'anon');
+  assert.equal(at('/api/openprofiles', 'Mozilla/5.0 niche-db/0.1'), 'anon');
+  // A session or a key still places the caller by what it carries.
+  const r = new Request('https://rssamplifier.com/api/openprofiles', {
+    headers: { 'user-agent': ua, cookie: 'rsa_session=abc' },
+  });
+  assert.equal(tierFor(r).name, 'session');
+  assert.ok(TIERS.house.burst >= 600, 'the house rung is at least 600 a minute');
+});