diff --git a/apps/web/package.json b/apps/web/package.json index 26402d8..a807abb 100644 --- a/apps/web/package.json +++ b/apps/web/package.json @@ -12,7 +12,7 @@ "dependencies": { "@profullstack/player": "^0.3.1", "@profullstack/rssamplifier": "workspace:*", - "@profullstack/x402-gateway": "0.3.0", + "@profullstack/x402-gateway": "0.4.0", "@rssamplifier/auth": "workspace:*", "@rssamplifier/db": "workspace:*", "@rssamplifier/feed": "workspace:*", diff --git a/apps/web/src/lib/crawl-gateway.js b/apps/web/src/lib/crawl-gateway.js index 1cde064..77d3499 100644 --- a/apps/web/src/lib/crawl-gateway.js +++ b/apps/web/src/lib/crawl-gateway.js @@ -1,4 +1,4 @@ -import { createGateway, isTrainingAgent, RETRIEVAL_AGENTS } from '@profullstack/x402-gateway'; +import { createGateway, isTrainingAgent, readPass, RETRIEVAL_AGENTS } from '@profullstack/x402-gateway'; import { crawlSales } from '@rssamplifier/db'; import { db } from './db.js'; @@ -186,6 +186,26 @@ export const gateway = createGateway({ */ chargeSpoofedBrowsers: true, exempt, + /* + * What a pass actually buys, printed on the sales page and in every 402. + * + * Every line is rate, and that is not a hedge — it is the directory's whole + * position, argued in lib/tiers.js: the data stays open to everyone, no key, + * no account, no field withheld, and money buys speed. A benefits list that + * promised fields or endpoints the free tier cannot have would be selling the + * opposite of what this site is for, and would cost us the search and AI + * answer traffic that is the only way a person finds us. + * + * Written as the numbers rather than as adjectives, because the reader is + * deciding between paying, waiting, and rotating addresses, and only the + * numbers settle that. + */ + benefits: [ + '120,000 requests an hour, up from 600 anonymous or 6,000 signed in', + '2,000 requests a minute of burst, up from 120', + 'No throttle on any page, feed, or API route, for the whole day', + 'Every field and every entry point stays open to everyone, paid or not: a pass buys rate, never access', + ], /* * Book the sale. * @@ -228,3 +248,44 @@ export const gateway = createGateway({ * @type {(request: Request) => Promise} */ export const gate = x402Proxy(gateway); + +/** Where a pass is presented: the named header, or a bearer token. */ +const PASS_HEADER = 'x-crawl-pass'; +const BEARER_PASS = /^Bearer\s+(cp_[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+)$/i; + +/** + * Whether this request carries a crawl pass that is genuinely ours and still + * live. + * + * The gate already reads the pass, and answers `undefined` for a holder exactly + * as it does for a person — which is correct for deciding whether to charge, + * and loses the one fact the throttle needs: that this caller has paid. Rather + * than have the gate report it, which would grow a contract five other sites + * depend on, the pass is read again here. It is an HMAC over a short string + * with no database behind it, so reading it twice costs a hash. + * + * The secret is the CoinPay key, which is what `createGateway` defaults to when + * no explicit `secret` is passed, so this verifies exactly the tokens the + * gateway mints and nothing else. With no key configured there are no valid + * passes to find, and every caller falls through to the free ladder. + * + * @param {Request} request + * @returns {Promise} + */ +export async function hasValidPass(request) { + const secret = env['COINPAY_X402_KEY']; + if (!secret) return false; + + const direct = request.headers.get(PASS_HEADER); + const bearer = BEARER_PASS.exec(request.headers.get('authorization') ?? ''); + const token = direct ? direct.trim() : bearer ? bearer[1] : null; + if (!token) return false; + + try { + return Boolean(await readPass(token, { secret })); + } catch { + // A malformed token is not an error worth failing a request over; it is + // simply not a pass, and the caller lands on the free ladder. + return false; + } +} diff --git a/apps/web/src/lib/tiers.js b/apps/web/src/lib/tiers.js index 9c7804b..1634ae7 100644 --- a/apps/web/src/lib/tiers.js +++ b/apps/web/src/lib/tiers.js @@ -101,6 +101,22 @@ export const TIERS = { hourly: FREE_HOURLY * AUTH_MULTIPLIER, }, sponsor: { name: 'sponsor', burst: envInt('TIER_SPONSOR_BURST', 2_000), hourly: SPONSOR_HOURLY }, + /** + * A bought crawl pass, at the sponsor's ceiling. + * + * The rung the ladder was missing. Until now the top of it was "ask us and we + * will sponsor you", which is not something a program can do at three in the + * morning, and the pass that /crawl already sold bought a training crawler + * past a 402 and then met the same 600-an-hour throttle as everyone else. So + * the thing we sell bought nothing a heavy caller could feel. + * + * Deliberately the same numbers as `sponsor` rather than higher: a sponsored + * key is a relationship and a pass is a dollar, and there is no honest reason + * the dollar should out-rank the relationship. The reasoning for that ceiling + * being bounded rather than infinite is in SPONSOR_HOURLY above and applies + * unchanged here. + */ + pass: { name: 'pass', burst: envInt('TIER_SPONSOR_BURST', 2_000), hourly: SPONSOR_HOURLY }, }; /** diff --git a/apps/web/src/proxy.js b/apps/web/src/proxy.js index 9953d8e..b7e362b 100644 --- a/apps/web/src/proxy.js +++ b/apps/web/src/proxy.js @@ -1,10 +1,10 @@ import { NextResponse } from 'next/server'; import { SIGNED_IN_HINT_COOKIE, hintToRestore } from './lib/session-hint.js'; -import { gate } from './lib/crawl-gateway.js'; +import { gate, hasValidPass } from './lib/crawl-gateway.js'; import { attempt, callerIdentity } from './lib/crawlThrottle.js'; import { countRequest } from './lib/trafficCounter.js'; -import { tierFor } from './lib/tiers.js'; +import { TIERS, tierFor } from './lib/tiers.js'; /** * The one thing that runs in front of every request. @@ -63,7 +63,19 @@ export async function proxy(request) { * unlimited, and nothing above it can be worth paying for. Signed in is now * a large budget rather than no budget. */ - const tier = tierFor(request); + /* + * A bought pass is the top rung, and it is checked before the rest of the + * ladder because it is the only one someone paid for. Without this the pass + * we sell at /crawl bought a crawler past the 402 and then met the same + * 600-an-hour throttle as an anonymous curl, which is to say it bought + * nothing anyone could feel. + * + * The gate above has already read the same token and said nothing about it, + * because "should this be charged" and "what allowance is this" are different + * questions. Reading it again is one HMAC, and only for a request that + * actually presents a token. + */ + const tier = (await hasValidPass(request)) ? TIERS.pass : tierFor(request); const verdict = attempt(callerIdentity(request), Date.now(), tier); @@ -112,6 +124,29 @@ function tooMany(verdict, tier) { ? 'Create an API key at https://rssamplifier.com/account and send it as a bearer token; a sponsored key raises the ceiling further.' : 'This is the sponsor ceiling. If you need more than this, ask and we will raise it.'; + /* + * The rung a program can climb on its own. + * + * Every branch above ends at something only a person can do: read an email, + * fill in a form, ask us. An agent that hits this wall at three in the + * morning has nowhere to go, and the honest options left to it are to slow + * down or to spread itself over a proxy pool. So the pass is named here, with + * its price, as the one upgrade that needs no human on either side. + * + * Said to the paid tier too, where it reads as "you already have this", + * because a caller at the sponsor ceiling asking what is above it should be + * told there is nothing rather than sold something twice. + */ + const buyable = + tier.name === 'pass' || tier.name === 'sponsor' + ? undefined + : { + url: 'https://rssamplifier.com/crawl', + price: '1.00 USD per day, USDC, settled by CoinPay', + buys: '120,000 requests an hour and 2,000 a minute, for the whole day', + how: 'Fetch https://rssamplifier.com/crawl with Accept: application/json for an x402 offer, pay it, then send the pass in the x-crawl-pass header. No account and no human needed.', + }; + return NextResponse.json( { error: 'rate limit exceeded', @@ -121,6 +156,7 @@ function tooMany(verdict, tier) { tier: tier.name, hourlyLimit: Number.isFinite(tier.hourly) ? tier.hourly : null, upgrade: nextRung, + ...(buyable ? { buy: buyable } : {}), retryAfter: verdict.retryAfter, }, { diff --git a/apps/web/test/paid-tier.test.js b/apps/web/test/paid-tier.test.js new file mode 100644 index 0000000..faee42a --- /dev/null +++ b/apps/web/test/paid-tier.test.js @@ -0,0 +1,122 @@ +import test from 'node:test'; +import assert from 'node:assert/strict'; + +import { mintPass } from '@profullstack/x402-gateway'; + +import { TIERS } from '../src/lib/tiers.js'; + +/** + * The pass rung, and the thing it fixes. + * + * Until now /crawl sold a pass that bought a training crawler past the 402 and + * then met the same 600-an-hour throttle as an anonymous curl. The pass is now + * a rung on the ladder, so the thing we sell buys something a caller can feel. + * + * `hasValidPass` is imported lazily inside each test: lib/crawl-gateway.js + * reads COINPAY_X402_KEY at module scope through `env`, and these set it first. + */ +const SECRET = 'cp_live_test_secret_0123456789'; + +async function passModule() { + process.env.COINPAY_X402_KEY = SECRET; + return import('../src/lib/crawl-gateway.js'); +} + +const withHeaders = (headers) => + new Request('https://rssamplifier.com/topics/anything', { headers: new Headers(headers) }); + +async function livePass() { + const now = Math.floor(Date.now() / 1000); + const { token } = await mintPass({ secret: SECRET, ref: 'test', expiresAt: now + 3600, now }); + return token; +} + +test('a pass sits at the sponsor ceiling, not above it', () => { + // A sponsored key is a relationship and a pass is a dollar. There is no + // honest reason the dollar should out-rank the relationship. + assert.equal(TIERS.pass.hourly, TIERS.sponsor.hourly); + assert.equal(TIERS.pass.burst, TIERS.sponsor.burst); +}); + +test('a pass is worth far more than signing in, or nobody would buy one', () => { + assert.ok( + TIERS.pass.hourly > TIERS.session.hourly, + 'a bought pass must beat the free account tier', + ); + assert.ok(TIERS.pass.hourly > TIERS.anon.hourly * 100); +}); + +test('a live pass in the header is recognised', async () => { + const { hasValidPass } = await passModule(); + const token = await livePass(); + assert.equal(await hasValidPass(withHeaders({ 'x-crawl-pass': token })), true); +}); + +test('a live pass presented as a bearer token is recognised', async () => { + // The gateway accepts both spellings, so the throttle must agree with it or + // a caller doing exactly what the sales page said gets throttled anyway. + const { hasValidPass } = await passModule(); + const token = await livePass(); + assert.equal(await hasValidPass(withHeaders({ authorization: `Bearer ${token}` })), true); +}); + +test('no pass is not a pass', async () => { + const { hasValidPass } = await passModule(); + assert.equal(await hasValidPass(withHeaders({})), false); +}); + +test('a forged pass is refused, which is the whole point of signing them', async () => { + const { hasValidPass } = await passModule(); + const token = await livePass(); + const tampered = `${token.slice(0, -1)}${token.endsWith('A') ? 'B' : 'A'}`; + assert.equal(await hasValidPass(withHeaders({ 'x-crawl-pass': tampered })), false); +}); + +test('a pass from a different secret is refused', async () => { + const { hasValidPass } = await passModule(); + const now = Math.floor(Date.now() / 1000); + const { token } = await mintPass({ + secret: 'someone-elses-key', + ref: null, + expiresAt: now + 3600, + now, + }); + assert.equal(await hasValidPass(withHeaders({ 'x-crawl-pass': token })), false); +}); + +test('an expired pass is refused', async () => { + const { hasValidPass } = await passModule(); + const now = Math.floor(Date.now() / 1000); + // Minted live, then judged from a moment after it lapsed. + const { token } = await mintPass({ secret: SECRET, ref: null, expiresAt: now + 1, now }); + await new Promise((resolve) => setTimeout(resolve, 1100)); + assert.equal(await hasValidPass(withHeaders({ 'x-crawl-pass': token })), false); +}); + +test('garbage in the header is not an error, just not a pass', async () => { + const { hasValidPass } = await passModule(); + for (const value of ['', ' ', 'cp_', 'cp_nodot', 'not-a-pass', 'Bearer nonsense']) { + assert.equal( + await hasValidPass(withHeaders({ 'x-crawl-pass': value })), + false, + JSON.stringify(value), + ); + } +}); + +test('the sales page lists what a pass buys, and every line of it is rate', async () => { + // The directory's position is that data is open and money buys speed. A + // benefits list promising fields or endpoints the free tier cannot have would + // sell the opposite of what this site is for. + const { gateway } = await passModule(); + const benefits = gateway.options.benefits; + assert.ok(Array.isArray(benefits) && benefits.length > 0, 'a pass has to say what it buys'); + assert.ok( + benefits.some((line) => /120,000 requests an hour/.test(line)), + 'the headline number belongs in the list', + ); + assert.ok( + benefits.some((line) => /rate, never access/.test(line)), + 'and so does the promise that nothing is withheld', + ); +}); diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index d473a7d..36a6495 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -56,8 +56,8 @@ importers: specifier: workspace:* version: link:../cli '@profullstack/x402-gateway': - specifier: 0.3.0 - version: 0.3.0 + specifier: 0.4.0 + version: 0.4.0 '@rssamplifier/auth': specifier: workspace:* version: link:../../packages/auth @@ -626,8 +626,8 @@ packages: engines: {node: '>=20.19'} hasBin: true - '@profullstack/x402-gateway@0.3.0': - resolution: {integrity: sha512-aaSMdtVInaAD6te/RnNnnqZW2+oo/dUsOTFTJCl0hOn58s85Yw3vZGk3KCtZmjrwV2O2rdTjKlZjbKp3CGgntw==} + '@profullstack/x402-gateway@0.4.0': + resolution: {integrity: sha512-Rk3dtpwgi5rOIcHvsRqv7XvbT8rEHyhN/d3zdtsPe8zQfhhYWEjuFuvd/2bIWoWZ6fILQRQOCbejOe2VCRlsNg==} engines: {node: '>=20.11'} '@simplewebauthn/browser@13.3.0': @@ -1384,7 +1384,7 @@ snapshots: '@noble/curves': 2.4.0 '@noble/hashes': 2.4.0 - '@profullstack/x402-gateway@0.3.0': {} + '@profullstack/x402-gateway@0.4.0': {} '@simplewebauthn/browser@13.3.0': {} diff --git a/pnpm-workspace.yaml b/pnpm-workspace.yaml index 5341416..9591678 100644 --- a/pnpm-workspace.yaml +++ b/pnpm-workspace.yaml @@ -13,7 +13,7 @@ allowBuilds: msgpackr-extract: false minimumReleaseAgeExclude: - '@profullstack/player@0.2.0 || 0.3.1' - - '@profullstack/x402-gateway@0.1.0 || 0.2.1 || 0.3.0' + - '@profullstack/x402-gateway@0.1.0 || 0.2.1 || 0.3.0 || 0.4.0' - '@profullstack/x402-client@0.2.0' - '@profullstack/leaderboard@0.3.0 || 0.3.1' - '@profullstack/partners@0.2.0'