feat: add search index headers and robots.txt generation for SEO control

src/index.ts

@@ -31,13 +31,33 @@ function isWorkerHandledPath(path: string): boolean {
   );
 }
 
+function addSearchIndexHeaders(request: Request, response: Response): Response {
+  const url = new URL(request.url);
+  const contentType = String(response.headers.get('Content-Type') || '').toLowerCase();
+  const shouldNoIndex =
+    url.pathname === '/robots.txt' ||
+    contentType.includes('text/html');
+
+  if (!shouldNoIndex) return response;
+
+  const headers = new Headers(response.headers);
+  headers.set('X-Robots-Tag', 'noindex, nofollow, noarchive, nosnippet');
+
+  return new Response(response.body, {
+    status: response.status,
+    statusText: response.statusText,
+    headers,
+  });
+}
+
 async function maybeServeAsset(request: Request, env: Env): Promise<Response | null> {
   if (!env.ASSETS) return null;
   if (request.method !== 'GET' && request.method !== 'HEAD') return null;
   const url = new URL(request.url);
   if (isWorkerHandledPath(url.pathname)) return null;
 
-  return env.ASSETS.fetch(request);
+  const response = await env.ASSETS.fetch(request);
+  return addSearchIndexHeaders(request, response);
 }
 
 async function ensureDatabaseInitialized(env: Env): Promise<void> {

webapp/vite.config.ts

@@ -5,12 +5,34 @@ import { defineConfig } from 'vite';
 
 const rootDir = fileURLToPath(new URL('.', import.meta.url));
 
+function searchIndexPolicyPlugin(isDemo: boolean) {
+  return {
+    name: 'nodewarden-search-index-policy',
+    transformIndexHtml(html: string) {
+      if (isDemo) return html;
+      return html.replace(
+        '<meta name="viewport" content="width=device-width, initial-scale=1.0" />',
+        '<meta name="viewport" content="width=device-width, initial-scale=1.0" />\n    <meta name="robots" content="noindex, nofollow, noarchive, nosnippet" />'
+      );
+    },
+    generateBundle() {
+      this.emitFile({
+        type: 'asset',
+        fileName: 'robots.txt',
+        source: isDemo
+          ? 'User-agent: *\nAllow: /\n'
+          : 'User-agent: *\nDisallow: /\n',
+      });
+    },
+  };
+}
+
 export default defineConfig(({ mode }) => {
   const isDemo = mode === 'demo';
 
   return {
     root: rootDir,
-    plugins: [preact()],
+    plugins: [preact(), searchIndexPolicyPlugin(isDemo)],
     define: {
       __NODEWARDEN_DEMO__: JSON.stringify(isDemo),
     },