Add server-side AI bot tracking to my Lovable site so I can see which AI assistants (ChatGPT, Claude, Perplexity, Gemini, Copilot, etc.) read my pages and which ones recommend me.
Replace the contents of src/server.ts with the code below. The tracking runs inside the TanStack Start fetch handler — every incoming request is checked against a list of known AI bot User-Agents, and matching visits are POSTed to xSeek (with the full page URL) before the request is handed to the existing server handler. The POST is handed to ctx.waitUntil() when the runtime provides it, and awaited otherwise so the runtime never drops the request. Keep the [xseek] console.log lines exactly as written — they show every step (request seen, bot matched, POST status) in the server logs so tracking can be verified end to end.
If src/server.ts already has custom logic beyond the default TanStack Start entry, preserve it. Just ensure the xSeek bot-tracking call runs FIRST inside the fetch handler, before any other request processing.
```ts
import "./lib/error-capture";
import { consumeLastCapturedError } from "./lib/error-capture";
import { renderErrorPage } from "./lib/error-page";
type ServerEntry = {
fetch: (request: Request, env: unknown, ctx: unknown) => Promise<Response> | Response;
};
interface Env {
XSEEK_API_KEY?: string;
XSEEK_WEBSITE_ID?: string;
}
// ── xSeek AI bot tracking ──────────────────────────────────────────────────
// Known AI bot User-Agent patterns. Synced from xSeek's public catalog —
// refresh from https://www.xseek.io/api/v1/robots when new crawlers ship.
const XSEEK_BOTS: { name: string; pattern: RegExp }[] = [
{ name: "anthropic-ai", pattern: /anthropic-ai/i },
{ name: "claudebot", pattern: /ClaudeBot/i },
{ name: "claude-web", pattern: /claude-web/i },
{ name: "claude-user", pattern: /Claude-User/i },
{ name: "claude-searchbot", pattern: /Claude-SearchBot/i },
{ name: "claude-code", pattern: /claude-code\//i },
{ name: "perplexitybot", pattern: /PerplexityBot/i },
{ name: "perplexity-user", pattern: /Perplexity-User/i },
{ name: "grokbot", pattern: /GrokBot(?!.*DeepSearch)/i },
{ name: "grok-search", pattern: /xAI-Grok/i },
{ name: "grok-deepsearch", pattern: /Grok-DeepSearch/i },
{ name: "GPTBot", pattern: /GPTBot/i },
{ name: "chatgpt-user", pattern: /ChatGPT-User/i },
{ name: "oai-searchbot", pattern: /OAI-SearchBot/i },
{ name: "google-extended", pattern: /Google-Extended/i },
{ name: "Google-Agent", pattern: /Google-Agent/i },
{ name: "applebot", pattern: /Applebot(?!-Extended)/i },
{ name: "applebot-extended", pattern: /Applebot-Extended/i },
{ name: "meta-external", pattern: /meta-externalagent/i },
{ name: "meta-externalfetcher", pattern: /meta-externalfetcher/i },
{ name: "bingbot", pattern: /Bingbot(?!.*AI)/i },
{ name: "bingpreview", pattern: /bingbot.*Chrome/i },
{ name: "microsoftpreview", pattern: /MicrosoftPreview/i },
{ name: "cohere-ai", pattern: /cohere-ai/i },
{ name: "cohere-training-data-crawler", pattern: /cohere-training-data-crawler/i },
{ name: "youbot", pattern: /YouBot/i },
{ name: "duckassistbot", pattern: /DuckAssistBot/i },
{ name: "semanticscholarbot", pattern: /SemanticScholarBot/i },
{ name: "ccbot", pattern: /CCBot/i },
{ name: "ai2bot", pattern: /AI2Bot/i },
{ name: "ai2bot-dolma", pattern: /AI2Bot-Dolma/i },
{ name: "aihitbot", pattern: /aiHitBot/i },
{ name: "amazonbot", pattern: /Amazonbot/i },
{ name: "novaact", pattern: /NovaAct/i },
{ name: "brightbot", pattern: /Brightbot/i },
{ name: "bytespider", pattern: /Bytespider/i },
{ name: "tiktokspider", pattern: /TikTokSpider/i },
{ name: "cotoyogi", pattern: /Cotoyogi/i },
{ name: "crawlspace", pattern: /Crawlspace/i },
{ name: "pangubot", pattern: /PanguBot/i },
{ name: "petalbot", pattern: /PetalBot/i },
{ name: "sidetrade-indexer", pattern: /Sidetrade indexer bot/i },
{ name: "timpibot", pattern: /Timpibot/i },
{ name: "omgili", pattern: /omgili/i },
{ name: "omgilibot", pattern: /omgilibot/i },
{ name: "webzio-extended", pattern: /Webzio-Extended/i },
{ name: "baiduspider", pattern: /Baiduspider/i },
{ name: "mistralai-user", pattern: /MistralAI-User/i },
];
async function trackXseekBot(request: Request, env: Env | undefined): Promise<void> {
const apiKey = env?.XSEEK_API_KEY ?? (typeof process !== "undefined" ? process.env?.XSEEK_API_KEY : undefined);
const websiteId =
env?.XSEEK_WEBSITE_ID ?? (typeof process !== "undefined" ? process.env?.XSEEK_WEBSITE_ID : undefined);
const ua = request.headers.get("user-agent") ?? "";
const url = new URL(request.url);
console.log(`[xseek] incoming ${request.method} ${url.pathname} ua="${ua.slice(0, 140)}"`);
if (!apiKey || !websiteId) {
console.warn(`[xseek] missing env (apiKey=${!!apiKey}, websiteId=${!!websiteId}) — skipping`);
return;
}
const bot = XSEEK_BOTS.find((b) => b.pattern.test(ua));
if (!bot) {
console.log(`[xseek] no bot pattern matched for ua="${ua.slice(0, 80)}" — skipping`);
return;
}
console.log(`[xseek] matched bot=${bot.name} → POST to xseek.io`);
const ip =
request.headers.get("cf-connecting-ip") ?? request.headers.get("x-forwarded-for")?.split(",")[0]?.trim() ?? "";
const endpoint = `https://www.xseek.io/api/v1/websites/${websiteId}/ai-visits`;
const payload = {
botName: bot.name,
userAgent: ua,
url: url.toString(),
ip,
referer: request.headers.get("referer") ?? "",
};
const started = Date.now();
try {
const res = await fetch(endpoint, {
method: "POST",
headers: {
Authorization: `Bearer ${apiKey}`,
"Content-Type": "application/json",
},
body: JSON.stringify(payload),
signal: AbortSignal.timeout(5000),
});
const ms = Date.now() - started;
const text = await res.text().catch(() => "<unreadable>");
if (!res.ok) {
console.error(`[xseek] POST ${res.status} ${res.statusText} in ${ms}ms — body: ${text.slice(0, 500)}`);
} else {
console.log(`[xseek] POST ok ${res.status} in ${ms}ms bot=${bot.name} — body: ${text.slice(0, 200)}`);
}
} catch (err) {
console.error(`[xseek] POST threw after ${Date.now() - started}ms:`, err);
}
}
// ────────────────────────────────────────────────────────────────────────────
let serverEntryPromise: Promise<ServerEntry> | undefined;
async function getServerEntry(): Promise<ServerEntry> {
if (!serverEntryPromise) {
serverEntryPromise = import("@tanstack/react-start/server-entry").then(
(m) => (m.default ?? m) as ServerEntry,
);
}
return serverEntryPromise;
}
async function normalizeCatastrophicSsrResponse(response: Response): Promise<Response> {
if (response.status < 500) return response;
const contentType = response.headers.get("content-type") ?? "";
if (!contentType.includes("application/json")) return response;
const body = await response.clone().text();
if (!body.includes('"unhandled":true') || !body.includes('"message":"HTTPError"')) {
return response;
}
console.error(consumeLastCapturedError() ?? new Error(`h3 swallowed SSR error: ${body}`));
return new Response(renderErrorPage(), {
status: 500,
headers: { "content-type": "text/html; charset=utf-8" },
});
}
export default {
async fetch(request: Request, env: unknown, ctx: unknown) {
const trackPromise = trackXseekBot(request, env as Env);
const waitUntil = (ctx as { waitUntil?: (p: Promise<unknown>) => void } | null)?.waitUntil;
if (typeof waitUntil === "function") {
console.log("[xseek] handing tracking to ctx.waitUntil");
waitUntil.call(ctx, trackPromise);
} else {
// No waitUntil (e.g. Node dev server) — await so the request actually completes.
console.log("[xseek] no ctx.waitUntil, awaiting tracking inline");
await trackPromise;
}
try {
const handler = await getServerEntry();
const response = await handler.fetch(request, env, ctx);
return await normalizeCatastrophicSsrResponse(response);
} catch (error) {
console.error(error);
return new Response(renderErrorPage(), {
status: 500,
headers: { "content-type": "text/html; charset=utf-8" },
});
}
},
};
```
Required environment variables (add in Lovable → Project Settings → Environment Variables):
- XSEEK_API_KEY — must include both articles:read and ai_visits:push privileges
- XSEEK_WEBSITE_ID — find it in your xSeek dashboard URL
After deploying, verify with a curl using a bot User-Agent:
curl -A "Mozilla/5.0 (compatible; GPTBot/1.0)" https://yoursite.com/
The visit should appear in xSeek → AI Visits within ~30 seconds.
