site: publish robots.txt with sitemap and content signals (#481)

The live robots.txt held only Cloudflare's content-signal preamble: no
user-agent group and no Sitemap line. Add an origin robots.txt that allows
all crawlers, states search=yes and ai-input=yes, and points at the sitemap
index. ai-train is left unstated. The site check now fails if the built
robots.txt loses the sitemap line.

Author: vishr

site/public/robots.txt

@@ -0,0 +1,5 @@
+User-agent: *
+Content-Signal: search=yes, ai-input=yes
+Allow: /
+
+Sitemap: https://echo.labstack.com/sitemap-index.xml

site/scripts/check-site.mjs

@@ -97,6 +97,9 @@ for (const locale of ['es', 'ja', 'pt-br', 'zh-cn']) {
 for (const path of ['sitemap-index.xml', 'pagefind/pagefind.js']) {
   if (!existsSync(join(distDir, path))) problems.push(`Missing ${path}`);
 }
+const robots = join(distDir, 'robots.txt');
+if (!existsSync(robots)) problems.push('Missing robots.txt');
+else if (!readFileSync(robots, 'utf8').includes('Sitemap: https://echo.labstack.com/sitemap-index.xml\n')) problems.push('robots.txt: missing sitemap');
 for (const path of ['llms.txt', 'next/llms.txt']) {
   const file = join(distDir, path);
   if (!existsSync(file)) { problems.push(`Missing ${path}`); continue; }

site/scripts/site-checks.test.mjs

@@ -32,6 +32,7 @@ test('new routes pass without accepting a baseline; removals and new-page broken
   for (const locale of ['es', 'ja', 'pt-br', 'zh-cn']) file(site, `dist/${locale}/middleware/index.html`, `<html lang="${locale}"></html>`);
   for (const path of ['sitemap-index.xml', 'pagefind/pagefind.js']) file(site, `dist/${path}`, '');
   for (const path of ['llms.txt', 'next/llms.txt']) file(site, `dist/${path}`, '# Echo\n');
+  file(site, 'dist/robots.txt', 'Sitemap: https://echo.labstack.com/sitemap-index.xml\n');
   file(site, 'dist/new-page/index.html', '<html lang="en"><a href="/">Home</a></html>');
   const accepted = check(site, 'check-site.mjs');
   assert.equal(accepted.status, 0, accepted.stderr);