diff --git a/src/index.ts b/src/index.ts index afe7402..1f3d4cf 100644 --- a/src/index.ts +++ b/src/index.ts @@ -31,13 +31,33 @@ function isWorkerHandledPath(path: string): boolean { ); } +function addSearchIndexHeaders(request: Request, response: Response): Response { + const url = new URL(request.url); + const contentType = String(response.headers.get('Content-Type') || '').toLowerCase(); + const shouldNoIndex = + url.pathname === '/robots.txt' || + contentType.includes('text/html'); + + if (!shouldNoIndex) return response; + + const headers = new Headers(response.headers); + headers.set('X-Robots-Tag', 'noindex, nofollow, noarchive, nosnippet'); + + return new Response(response.body, { + status: response.status, + statusText: response.statusText, + headers, + }); +} + async function maybeServeAsset(request: Request, env: Env): Promise { if (!env.ASSETS) return null; if (request.method !== 'GET' && request.method !== 'HEAD') return null; const url = new URL(request.url); if (isWorkerHandledPath(url.pathname)) return null; - return env.ASSETS.fetch(request); + const response = await env.ASSETS.fetch(request); + return addSearchIndexHeaders(request, response); } async function ensureDatabaseInitialized(env: Env): Promise { diff --git a/webapp/vite.config.ts b/webapp/vite.config.ts index 5d2226b..7f18fc5 100644 --- a/webapp/vite.config.ts +++ b/webapp/vite.config.ts @@ -5,12 +5,34 @@ import { defineConfig } from 'vite'; const rootDir = fileURLToPath(new URL('.', import.meta.url)); +function searchIndexPolicyPlugin(isDemo: boolean) { + return { + name: 'nodewarden-search-index-policy', + transformIndexHtml(html: string) { + if (isDemo) return html; + return html.replace( + '', + '\n ' + ); + }, + generateBundle() { + this.emitFile({ + type: 'asset', + fileName: 'robots.txt', + source: isDemo + ? 'User-agent: *\nAllow: /\n' + : 'User-agent: *\nDisallow: /\n', + }); + }, + }; +} + export default defineConfig(({ mode }) => { const isDemo = mode === 'demo'; return { root: rootDir, - plugins: [preact()], + plugins: [preact(), searchIndexPolicyPlugin(isDemo)], define: { __NODEWARDEN_DEMO__: JSON.stringify(isDemo), },