ai-cs/frontend/app/robots.ts

import type { MetadataRoute } from "next";
import { getSiteUrl } from "@/lib/site";

export default function robots(): MetadataRoute.Robots {
  const base = getSiteUrl();

  return {
    rules: {
      userAgent: "*",
      allow: "/",
      disallow: ["/agent/", "/api/"],
    },
    sitemap: `${base}/sitemap.xml`,
  };
}