diff --git a/apps/web/package.json b/apps/web/package.json index 1230473..26402d8 100644 --- a/apps/web/package.json +++ b/apps/web/package.json @@ -27,6 +27,7 @@ "next": "^16.2.4", "react": "^19.2.0", "react-dom": "^19.2.0", - "@profullstack/leaderboard": "^0.3.0" + "@profullstack/leaderboard": "^0.3.0", + "@profullstack/partners": "^0.2.0" } } diff --git a/apps/web/src/app/sell/[[...path]]/route.js b/apps/web/src/app/sell/[[...path]]/route.js new file mode 100644 index 0000000..2b0da6e --- /dev/null +++ b/apps/web/src/app/sell/[[...path]]/route.js @@ -0,0 +1,20 @@ +import { partners } from '../../../lib/partners.js'; + +/** + * The seller side: where a publisher signs up, proves they own the site behind + * a feed we index, and gets paid a share of what crawlers pay for access. + * + * 404 when PARTNER_VERIFY_SECRET is unset, because the programme cannot run + * safely without it and a half-working signup is worse than none. + */ +export const dynamic = 'force-dynamic'; + +async function handle(request) { + const program = partners(); + if (!program) return new Response('Not found', { status: 404 }); + return (await program.handle(request)) ?? new Response('Not found', { status: 404 }); +} + +export const GET = handle; +export const POST = handle; +export const HEAD = handle; diff --git a/apps/web/src/lib/crawl-gateway.js b/apps/web/src/lib/crawl-gateway.js index aa889b3..1cde064 100644 --- a/apps/web/src/lib/crawl-gateway.js +++ b/apps/web/src/lib/crawl-gateway.js @@ -2,6 +2,7 @@ import { createGateway, isTrainingAgent, RETRIEVAL_AGENTS } from '@profullstack/ import { crawlSales } from '@rssamplifier/db'; import { db } from './db.js'; +import { splitSale } from './partners.js'; import { classifyAgent } from './traffic.js'; import { x402Proxy } from '@profullstack/x402-gateway/next'; @@ -75,6 +76,11 @@ export const OPEN_PATHS = [ // buying a pass. '/leaderboard', '/leaderboard/', + // The pitch is how a publisher finds out they can be paid for what a crawler + // is already taking. Charging the crawler to read our own recruiting page + // would be an odd way to run a marketplace. + '/sell', + '/sell/', ]; /** @@ -206,6 +212,12 @@ export const gateway = createGateway({ agent: classifyAgent(sale.userAgent), expiresAt: sale.expiresAt, }) + .then(() => + // Pay the publishers whose blogs were in the crawl. After the sale is + // booked, and never able to fail it: the money has already moved, and + // a split we can retry beats a 500 to a paying customer. + splitSale(sale).catch((err) => console.error('[partners] could not split the sale', err)), + ) .catch((err) => console.error('[x402] could not record the sale', err)), }); diff --git a/apps/web/src/lib/partners.js b/apps/web/src/lib/partners.js new file mode 100644 index 0000000..bec8bc5 --- /dev/null +++ b/apps/web/src/lib/partners.js @@ -0,0 +1,132 @@ +import { SESSION_COOKIE, resolveSession } from '@rssamplifier/auth'; +import { createPartners, sqlStore } from '@profullstack/partners'; + +import { db, siteUrl } from './db.js'; + +/** + * The seller side of the directory. + * + * Every blog here belongs to somebody else, and traffic_hourly says most of + * what reads them is machines. crawl_sales says what those machines paid. + * This is how the publishers get their share: prove you own the site behind + * a feed we already index, say which topics you write about, and take a cut. + * + * The module owns none of the auth. It asks who is here and we answer from + * the same session cookie the rest of the site uses. + */ + +const execute = async ({ sql, args = [] }) => db().execute({ sql, args }); +const store = sqlStore({ execute }); + +/** The session cookie off a bare Request; the module hands us one, not a Next context. */ +function cookieToken(request) { + const header = request.headers.get('cookie') ?? ''; + for (const part of header.split(';')) { + const eq = part.indexOf('='); + if (eq === -1) continue; + if (part.slice(0, eq).trim() === SESSION_COOKIE) return part.slice(eq + 1).trim(); + } + return null; +} + +async function currentUser(request) { + const token = cookieToken(request); + if (!token) return null; + try { + const user = await resolveSession(db(), token); + return user ? { id: String(user.id), name: user.name ?? user.email ?? null, email: user.email ?? null } : null; + } catch { + // A database hiccup degrades to "signed out" rather than a 500 on a page + // whose whole job is to explain the programme. + return null; + } +} + +/** + * The niches a partner may claim: the directory's own topics, busiest first. + * Resolved per request, so a topic that appeared this week is claimable this + * week. Capped, because the full list is thousands long and a checkbox for + * each is not a form anybody fills in. + */ +async function niches() { + const { rows } = await db().execute({ + sql: 'select slug from topics order by feed_count desc, slug limit 40', + args: [], + }); + return rows.map((r) => String(r.slug)); +} + +/** @type {ReturnType | null} */ +let program = null; + +/** + * Built lazily and only when the secret exists. + * + * The module refuses to construct without one, and it is right to: a + * guessable verification token pays the wrong person for someone else's + * writing. That refusal must not take the site down on a deploy where the + * variable was forgotten, so /sell simply does not exist until it is set. + */ +export function partners() { + if (program) return program; + const secret = process.env['PARTNER_VERIFY_SECRET']; + if (!secret) return null; + program = createPartners({ + siteName: 'RSS Amplifier', + siteUrl: siteUrl(), + basePath: '/sell', + store, + secret, + loginUrl: '/signin?next=/sell', + currentUser, + niches, + }); + return program; +} + +/** + * Split one crawl sale across the publishers whose blogs were in the crawl. + * + * Attribution belongs here because only this side knows whose feeds are in + * the index. A verified domain is matched against the host of each feed's + * site_url, and the sale is shared pro-rata by how many items that publisher + * has contributed, each at their own rate. + * + * The credit ref carries the partner id, so a settlement delivered twice pays + * once. Never allowed to fail the sale: the money has already moved. + */ +export async function splitSale(sale) { + const p = partners(); + if (!p || !sale?.ref || !Number(sale.totalCents)) return 0; + + const { rows } = await db().execute({ + sql: `select pp.partner_id as partner_id, count(fi.id) as items + from partner_properties pp + join feeds f + on f.site_url is not null + and ( + lower(replace(replace(replace(f.site_url, 'https://', ''), 'http://', ''), 'www.', '')) = pp.domain + or lower(replace(replace(replace(f.site_url, 'https://', ''), 'http://', ''), 'www.', '')) like pp.domain || '/%' + ) + join feed_items fi on fi.feed_id = f.id + where pp.verified_at is not null + group by pp.partner_id + having count(fi.id) > 0`, + args: [], + }); + if (!rows.length) return 0; + + const total = rows.reduce((n, r) => n + Number(r.items), 0); + let paid = 0; + for (const row of rows) { + const partnerId = String(row.partner_id); + const partner = await store.getPartnerById(partnerId); + if (!partner) continue; + const properties = await store.listProperties(partnerId); + const rate = p.rateFor(partner, properties); + const share = Math.floor((Number(sale.totalCents) * (Number(row.items) / total) * rate) / 100); + if (share <= 0) continue; + if (await p.credit({ partnerId, cents: share, ref: `${sale.ref}:${partnerId}` })) paid += share; + } + return paid; +} diff --git a/packages/db/migrations/20260906044000_partners.sql b/packages/db/migrations/20260906044000_partners.sql new file mode 100644 index 0000000..e45e4ac --- /dev/null +++ b/packages/db/migrations/20260906044000_partners.sql @@ -0,0 +1,44 @@ +-- The seller side: who may be paid for the writing in this directory. +-- +-- The directory is other people's blogs, and traffic_hourly says most of what +-- reads them is machines. crawl_sales says what those machines paid. These +-- three tables are the missing half: which of those publishers has proven a +-- domain, and what share of that money is theirs (@profullstack/partners). +-- +-- The DDL matches the package's own sqlStore().schema, and lives here as a +-- numbered migration so it arrives the way every other schema change does. +create table if not exists partner_accounts ( + id integer primary key autoincrement, + user_id text not null unique, + name text, + -- A comma-joined list, the package's storage shape. Nothing here parses it + -- by hand; every read goes through the package. + niches text not null default '', + payout_address text, + created_at integer not null default (cast(strftime('%s','now') as integer) * 1000) +); + +-- `domain` is unique across every partner rather than per partner: two +-- accounts claiming one site is the shape of somebody being paid for another +-- person's writing, so the database refuses it rather than the application +-- remembering to. +create table if not exists partner_properties ( + id integer primary key autoincrement, + partner_id text not null, + domain text not null unique, + verified_at integer, + method text, + created_at integer not null default (cast(strftime('%s','now') as integer) * 1000) +); +create index if not exists partner_properties_partner on partner_properties (partner_id); + +-- `ref` is unique so a settlement delivered twice pays once. Credits outlive +-- the property that earned them: removing a site does not erase its earnings. +create table if not exists partner_credits ( + id integer primary key autoincrement, + partner_id text not null, + cents integer not null, + ref text unique, + at integer not null +); +create index if not exists partner_credits_partner on partner_credits (partner_id); diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index b48a2bb..d473a7d 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -46,6 +46,9 @@ importers: '@profullstack/leaderboard': specifier: ^0.3.0 version: 0.3.0 + '@profullstack/partners': + specifier: ^0.2.0 + version: 0.2.0 '@profullstack/player': specifier: ^0.3.1 version: 0.3.1(react@19.2.8) @@ -601,6 +604,14 @@ packages: resolution: {integrity: sha512-SKPmIqhmrAhFQqHl9wJHV1NzJHUCNBsqWc1F6M5AeFfphXWGfKSmgeCC3QN37JH/cQ1NxS8kYC/DqbjQqNBoaQ==} engines: {node: '>=20.11'} + '@profullstack/leaderboard@0.3.1': + resolution: {integrity: sha512-LbxLwy+RvT/Qez0mfWCk4o9hvcq0kzwBZ0vF+CYp+oGvyqJWUaSTLhCHWGtKKAI3c0ZUXj6ll0iuRjbuZUE2Sg==} + engines: {node: '>=20.11'} + + '@profullstack/partners@0.2.0': + resolution: {integrity: sha512-h8Fal+ZMqpR2xixo8OC/Xb/OXfLXmt2BdBIgohC3tK6U3AkAsDrkUyrlf+DEF8Ut8mptDEjCWDz5EV/dIXBRHw==} + engines: {node: '>=20.11'} + '@profullstack/player@0.3.1': resolution: {integrity: sha512-/BvjRREIQ+WBw+MJXuSUDONreRnivB36wZDiflnzqrTtDhBUZEbkAgsfKF/m67U7GdSYl1uRJMbQoPEw5I9nMA==} engines: {node: '>=20'} @@ -1355,6 +1366,12 @@ snapshots: '@profullstack/leaderboard@0.3.0': {} + '@profullstack/leaderboard@0.3.1': {} + + '@profullstack/partners@0.2.0': + dependencies: + '@profullstack/leaderboard': 0.3.1 + '@profullstack/player@0.3.1(react@19.2.8)': dependencies: hls.js: 1.7.1 diff --git a/pnpm-workspace.yaml b/pnpm-workspace.yaml index e595706..5341416 100644 --- a/pnpm-workspace.yaml +++ b/pnpm-workspace.yaml @@ -15,4 +15,5 @@ minimumReleaseAgeExclude: - '@profullstack/player@0.2.0 || 0.3.1' - '@profullstack/x402-gateway@0.1.0 || 0.2.1 || 0.3.0' - '@profullstack/x402-client@0.2.0' - - '@profullstack/leaderboard@0.3.0' + - '@profullstack/leaderboard@0.3.0 || 0.3.1' + - '@profullstack/partners@0.2.0'