The pass we sell is now a rung on the ladder

apps/web/package.json

@@ -12,7 +12,7 @@
   "dependencies": {
     "@profullstack/player": "^0.3.1",
     "@profullstack/rssamplifier": "workspace:*",
-    "@profullstack/x402-gateway": "0.3.0",
+    "@profullstack/x402-gateway": "0.4.0",
     "@rssamplifier/auth": "workspace:*",
     "@rssamplifier/db": "workspace:*",
     "@rssamplifier/feed": "workspace:*",

apps/web/src/lib/crawl-gateway.js

@@ -1,4 +1,4 @@
-import { createGateway, isTrainingAgent, RETRIEVAL_AGENTS } from '@profullstack/x402-gateway';
+import { createGateway, isTrainingAgent, readPass, RETRIEVAL_AGENTS } from '@profullstack/x402-gateway';
 import { crawlSales } from '@rssamplifier/db';
 
 import { db } from './db.js';
@@ -186,6 +186,26 @@ export const gateway = createGateway({
    */
   chargeSpoofedBrowsers: true,
   exempt,
+  /*
+   * What a pass actually buys, printed on the sales page and in every 402.
+   *
+   * Every line is rate, and that is not a hedge — it is the directory's whole
+   * position, argued in lib/tiers.js: the data stays open to everyone, no key,
+   * no account, no field withheld, and money buys speed. A benefits list that
+   * promised fields or endpoints the free tier cannot have would be selling the
+   * opposite of what this site is for, and would cost us the search and AI
+   * answer traffic that is the only way a person finds us.
+   *
+   * Written as the numbers rather than as adjectives, because the reader is
+   * deciding between paying, waiting, and rotating addresses, and only the
+   * numbers settle that.
+   */
+  benefits: [
+    '120,000 requests an hour, up from 600 anonymous or 6,000 signed in',
+    '2,000 requests a minute of burst, up from 120',
+    'No throttle on any page, feed, or API route, for the whole day',
+    'Every field and every entry point stays open to everyone, paid or not: a pass buys rate, never access',
+  ],
   /*
    * Book the sale.
    *
@@ -228,3 +248,44 @@ export const gateway = createGateway({
  * @type {(request: Request) => Promise<Response | undefined>}
  */
 export const gate = x402Proxy(gateway);
+
+/** Where a pass is presented: the named header, or a bearer token. */
+const PASS_HEADER = 'x-crawl-pass';
+const BEARER_PASS = /^Bearer\s+(cp_[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+)$/i;
+
+/**
+ * Whether this request carries a crawl pass that is genuinely ours and still
+ * live.
+ *
+ * The gate already reads the pass, and answers `undefined` for a holder exactly
+ * as it does for a person — which is correct for deciding whether to charge,
+ * and loses the one fact the throttle needs: that this caller has paid. Rather
+ * than have the gate report it, which would grow a contract five other sites
+ * depend on, the pass is read again here. It is an HMAC over a short string
+ * with no database behind it, so reading it twice costs a hash.
+ *
+ * The secret is the CoinPay key, which is what `createGateway` defaults to when
+ * no explicit `secret` is passed, so this verifies exactly the tokens the
+ * gateway mints and nothing else. With no key configured there are no valid
+ * passes to find, and every caller falls through to the free ladder.
+ *
+ * @param {Request} request
+ * @returns {Promise<boolean>}
+ */
+export async function hasValidPass(request) {
+  const secret = env['COINPAY_X402_KEY'];
+  if (!secret) return false;
+
+  const direct = request.headers.get(PASS_HEADER);
+  const bearer = BEARER_PASS.exec(request.headers.get('authorization') ?? '');
+  const token = direct ? direct.trim() : bearer ? bearer[1] : null;
+  if (!token) return false;
+
+  try {
+    return Boolean(await readPass(token, { secret }));
+  } catch {
+    // A malformed token is not an error worth failing a request over; it is
+    // simply not a pass, and the caller lands on the free ladder.
+    return false;
+  }
+}

apps/web/src/lib/tiers.js

@@ -101,6 +101,22 @@ export const TIERS = {
     hourly: FREE_HOURLY * AUTH_MULTIPLIER,
   },
   sponsor: { name: 'sponsor', burst: envInt('TIER_SPONSOR_BURST', 2_000), hourly: SPONSOR_HOURLY },
+  /**
+   * A bought crawl pass, at the sponsor's ceiling.
+   *
+   * The rung the ladder was missing. Until now the top of it was "ask us and we
+   * will sponsor you", which is not something a program can do at three in the
+   * morning, and the pass that /crawl already sold bought a training crawler
+   * past a 402 and then met the same 600-an-hour throttle as everyone else. So
+   * the thing we sell bought nothing a heavy caller could feel.
+   *
+   * Deliberately the same numbers as `sponsor` rather than higher: a sponsored
+   * key is a relationship and a pass is a dollar, and there is no honest reason
+   * the dollar should out-rank the relationship. The reasoning for that ceiling
+   * being bounded rather than infinite is in SPONSOR_HOURLY above and applies
+   * unchanged here.
+   */
+  pass: { name: 'pass', burst: envInt('TIER_SPONSOR_BURST', 2_000), hourly: SPONSOR_HOURLY },
 };
 
 /**

apps/web/src/proxy.js

@@ -1,10 +1,10 @@
 import { NextResponse } from 'next/server';
 
 import { SIGNED_IN_HINT_COOKIE, hintToRestore } from './lib/session-hint.js';
-import { gate } from './lib/crawl-gateway.js';
+import { gate, hasValidPass } from './lib/crawl-gateway.js';
 import { attempt, callerIdentity } from './lib/crawlThrottle.js';
 import { countRequest } from './lib/trafficCounter.js';
-import { tierFor } from './lib/tiers.js';
+import { TIERS, tierFor } from './lib/tiers.js';
 
 /**
  * The one thing that runs in front of every request.
@@ -63,7 +63,19 @@ export async function proxy(request) {
    * unlimited, and nothing above it can be worth paying for. Signed in is now
    * a large budget rather than no budget.
    */
-  const tier = tierFor(request);
+  /*
+   * A bought pass is the top rung, and it is checked before the rest of the
+   * ladder because it is the only one someone paid for. Without this the pass
+   * we sell at /crawl bought a crawler past the 402 and then met the same
+   * 600-an-hour throttle as an anonymous curl, which is to say it bought
+   * nothing anyone could feel.
+   *
+   * The gate above has already read the same token and said nothing about it,
+   * because "should this be charged" and "what allowance is this" are different
+   * questions. Reading it again is one HMAC, and only for a request that
+   * actually presents a token.
+   */
+  const tier = (await hasValidPass(request)) ? TIERS.pass : tierFor(request);
 
   const verdict = attempt(callerIdentity(request), Date.now(), tier);
 
@@ -112,6 +124,29 @@ function tooMany(verdict, tier) {
         ? 'Create an API key at https://rssamplifier.com/account and send it as a bearer token; a sponsored key raises the ceiling further.'
         : 'This is the sponsor ceiling. If you need more than this, ask and we will raise it.';
 
+  /*
+   * The rung a program can climb on its own.
+   *
+   * Every branch above ends at something only a person can do: read an email,
+   * fill in a form, ask us. An agent that hits this wall at three in the
+   * morning has nowhere to go, and the honest options left to it are to slow
+   * down or to spread itself over a proxy pool. So the pass is named here, with
+   * its price, as the one upgrade that needs no human on either side.
+   *
+   * Said to the paid tier too, where it reads as "you already have this",
+   * because a caller at the sponsor ceiling asking what is above it should be
+   * told there is nothing rather than sold something twice.
+   */
+  const buyable =
+    tier.name === 'pass' || tier.name === 'sponsor'
+      ? undefined
+      : {
+          url: 'https://rssamplifier.com/crawl',
+          price: '1.00 USD per day, USDC, settled by CoinPay',
+          buys: '120,000 requests an hour and 2,000 a minute, for the whole day',
+          how: 'Fetch https://rssamplifier.com/crawl with Accept: application/json for an x402 offer, pay it, then send the pass in the x-crawl-pass header. No account and no human needed.',
+        };
+
   return NextResponse.json(
     {
       error: 'rate limit exceeded',
@@ -121,6 +156,7 @@ function tooMany(verdict, tier) {
       tier: tier.name,
       hourlyLimit: Number.isFinite(tier.hourly) ? tier.hourly : null,
       upgrade: nextRung,
+      ...(buyable ? { buy: buyable } : {}),
       retryAfter: verdict.retryAfter,
     },
     {

apps/web/test/paid-tier.test.js

@@ -0,0 +1,122 @@
+import test from 'node:test';
+import assert from 'node:assert/strict';
+
+import { mintPass } from '@profullstack/x402-gateway';
+
+import { TIERS } from '../src/lib/tiers.js';
+
+/**
+ * The pass rung, and the thing it fixes.
+ *
+ * Until now /crawl sold a pass that bought a training crawler past the 402 and
+ * then met the same 600-an-hour throttle as an anonymous curl. The pass is now
+ * a rung on the ladder, so the thing we sell buys something a caller can feel.
+ *
+ * `hasValidPass` is imported lazily inside each test: lib/crawl-gateway.js
+ * reads COINPAY_X402_KEY at module scope through `env`, and these set it first.
+ */
+const SECRET = 'cp_live_test_secret_0123456789';
+
+async function passModule() {
+  process.env.COINPAY_X402_KEY = SECRET;
+  return import('../src/lib/crawl-gateway.js');
+}
+
+const withHeaders = (headers) =>
+  new Request('https://rssamplifier.com/topics/anything', { headers: new Headers(headers) });
+
+async function livePass() {
+  const now = Math.floor(Date.now() / 1000);
+  const { token } = await mintPass({ secret: SECRET, ref: 'test', expiresAt: now + 3600, now });
+  return token;
+}
+
+test('a pass sits at the sponsor ceiling, not above it', () => {
+  // A sponsored key is a relationship and a pass is a dollar. There is no
+  // honest reason the dollar should out-rank the relationship.
+  assert.equal(TIERS.pass.hourly, TIERS.sponsor.hourly);
+  assert.equal(TIERS.pass.burst, TIERS.sponsor.burst);
+});
+
+test('a pass is worth far more than signing in, or nobody would buy one', () => {
+  assert.ok(
+    TIERS.pass.hourly > TIERS.session.hourly,
+    'a bought pass must beat the free account tier',
+  );
+  assert.ok(TIERS.pass.hourly > TIERS.anon.hourly * 100);
+});
+
+test('a live pass in the header is recognised', async () => {
+  const { hasValidPass } = await passModule();
+  const token = await livePass();
+  assert.equal(await hasValidPass(withHeaders({ 'x-crawl-pass': token })), true);
+});
+
+test('a live pass presented as a bearer token is recognised', async () => {
+  // The gateway accepts both spellings, so the throttle must agree with it or
+  // a caller doing exactly what the sales page said gets throttled anyway.
+  const { hasValidPass } = await passModule();
+  const token = await livePass();
+  assert.equal(await hasValidPass(withHeaders({ authorization: `Bearer ${token}` })), true);
+});
+
+test('no pass is not a pass', async () => {
+  const { hasValidPass } = await passModule();
+  assert.equal(await hasValidPass(withHeaders({})), false);
+});
+
+test('a forged pass is refused, which is the whole point of signing them', async () => {
+  const { hasValidPass } = await passModule();
+  const token = await livePass();
+  const tampered = `${token.slice(0, -1)}${token.endsWith('A') ? 'B' : 'A'}`;
+  assert.equal(await hasValidPass(withHeaders({ 'x-crawl-pass': tampered })), false);
+});
+
+test('a pass from a different secret is refused', async () => {
+  const { hasValidPass } = await passModule();
+  const now = Math.floor(Date.now() / 1000);
+  const { token } = await mintPass({
+    secret: 'someone-elses-key',
+    ref: null,
+    expiresAt: now + 3600,
+    now,
+  });
+  assert.equal(await hasValidPass(withHeaders({ 'x-crawl-pass': token })), false);
+});
+
+test('an expired pass is refused', async () => {
+  const { hasValidPass } = await passModule();
+  const now = Math.floor(Date.now() / 1000);
+  // Minted live, then judged from a moment after it lapsed.
+  const { token } = await mintPass({ secret: SECRET, ref: null, expiresAt: now + 1, now });
+  await new Promise((resolve) => setTimeout(resolve, 1100));
+  assert.equal(await hasValidPass(withHeaders({ 'x-crawl-pass': token })), false);
+});
+
+test('garbage in the header is not an error, just not a pass', async () => {
+  const { hasValidPass } = await passModule();
+  for (const value of ['', '   ', 'cp_', 'cp_nodot', 'not-a-pass', 'Bearer nonsense']) {
+    assert.equal(
+      await hasValidPass(withHeaders({ 'x-crawl-pass': value })),
+      false,
+      JSON.stringify(value),
+    );
+  }
+});
+
+test('the sales page lists what a pass buys, and every line of it is rate', async () => {
+  // The directory's position is that data is open and money buys speed. A
+  // benefits list promising fields or endpoints the free tier cannot have would
+  // sell the opposite of what this site is for.
+  const { gateway } = await passModule();
+  const benefits = gateway.options.benefits;
+  assert.ok(Array.isArray(benefits) && benefits.length > 0, 'a pass has to say what it buys');
+  assert.ok(
+    benefits.some((line) => /120,000 requests an hour/.test(line)),
+    'the headline number belongs in the list',
+  );
+  assert.ok(
+    benefits.some((line) => /rate, never access/.test(line)),
+    'and so does the promise that nothing is withheld',
+  );
+});

pnpm-workspace.yaml

@@ -13,7 +13,7 @@ allowBuilds:
   msgpackr-extract: false
 minimumReleaseAgeExclude:
   - '@profullstack/player@0.2.0 || 0.3.1'
-  - '@profullstack/x402-gateway@0.1.0 || 0.2.1 || 0.3.0'
+  - '@profullstack/x402-gateway@0.1.0 || 0.2.1 || 0.3.0 || 0.4.0'
   - '@profullstack/x402-client@0.2.0'
   - '@profullstack/leaderboard@0.3.0 || 0.3.1'
   - '@profullstack/partners@0.2.0'