User-agent: * Allow: / Allow: /api/public/ # Primary indexable surfaces — stated explicitly so no later pattern # accidentally withholds them from crawl budget. Allow: /doctor/ Allow: /cities/ Allow: /specialists/ Allow: /sitemap-cities.xml Allow: /sitemap-specialists.xml Disallow: /admin/ Disallow: /api/ Disallow: /checkout Disallow: /dashboard Disallow: /signin Disallow: /reset-password # /find-a-doctor results: the bare URL is the canonical, indexable hub. # Parameterized variants (?q=, ?specialty=, ?location=, ?page=, ?sort=, # tracking params) all render # and a canonical pointing back to /find-a-doctor, so they consolidate # rather than dilute. We intentionally do NOT Disallow the query string — # blocking it would hide the noindex+canonical signals from crawlers. # Block only the known-noise param families so budget is spent on real # results pages. Disallow: /find-a-doctor?*page= Disallow: /find-a-doctor?*sort= Disallow: /find-a-doctor?*utm_ Disallow: /find-a-doctor?*fbclid= Disallow: /find-a-doctor?*gclid= Disallow: /find-a-doctor-near-me?*page= Disallow: /find-a-doctor-near-me?*utm_ Disallow: /find-a-doctor-near-me?*fbclid= Disallow: /find-a-doctor-near-me?*gclid= # Explicitly allow AI / LLM crawlers so the site is eligible for # answer-engine citations (ChatGPT search, Claude, Perplexity, Google AI # Overviews, Bing/Copilot, Common Crawl-derived training corpora). User-agent: GPTBot Allow: / User-agent: OAI-SearchBot Allow: / User-agent: ChatGPT-User Allow: / User-agent: ClaudeBot Allow: / User-agent: Claude-Web Allow: / User-agent: anthropic-ai Allow: / User-agent: PerplexityBot Allow: / User-agent: Perplexity-User Allow: / User-agent: Google-Extended Allow: / User-agent: Googlebot Allow: / User-agent: Bingbot Allow: / User-agent: Applebot Allow: / User-agent: Applebot-Extended Allow: / User-agent: CCBot Allow: / User-agent: Meta-ExternalAgent Allow: / User-agent: Amazonbot Allow: / User-agent: YouBot Allow: / User-agent: DuckAssistBot Allow: / User-agent: cohere-ai Allow: / User-agent: Mistral-AI-Bot Allow: / # --------------------------------------------------------------------------- # Open crawl policy: previously-restricted scrapers, SEO tools, dataset # harvesters, site rippers, and generic HTTP libraries are now explicitly # ALLOWED to read and index the site. The server-side gate in src/start.ts # still protects against abusive burst / monetization-sweep patterns. # Enforcement contact: abuse@helpmefindadoctor.com. # --------------------------------------------------------------------------- User-agent: AhrefsBot Allow: / User-agent: SemrushBot Allow: / User-agent: MJ12bot Allow: / User-agent: DotBot Allow: / User-agent: rogerbot Allow: / User-agent: BLEXBot Allow: / User-agent: DataForSeoBot Allow: / User-agent: SeekportBot Allow: / User-agent: SerpstatBot Allow: / User-agent: LinkpadBot Allow: / User-agent: MegaIndex.ru Allow: / User-agent: PetalBot Allow: / User-agent: Bytespider Allow: / User-agent: ImagesiftBot Allow: / User-agent: Diffbot Allow: / User-agent: SiteAuditBot Allow: / User-agent: SEOkicks Allow: / User-agent: BUbiNG Allow: / User-agent: Screaming Frog SEO Spider Allow: / User-agent: SiteCheckerBotCrawler Allow: / User-agent: HTTrack Allow: / User-agent: WebCopier Allow: / User-agent: WebZIP Allow: / User-agent: Wget Allow: / User-agent: curl Allow: / User-agent: python-requests Allow: / User-agent: python-urllib Allow: / User-agent: Scrapy Allow: / User-agent: Node-Fetch Allow: / User-agent: axios Allow: / User-agent: Go-http-client Allow: / User-agent: Java Allow: / User-agent: Apache-HttpClient Allow: / User-agent: libwww-perl Allow: / User-agent: PhantomJS Allow: / User-agent: HeadlessChrome Allow: / User-agent: SputnikBot Allow: / User-agent: 360Spider Allow: / User-agent: YisouSpider Allow: / User-agent: MauiBot Allow: / User-agent: SurdotlyBot Allow: / User-agent: TurnitinBot Allow: / User-agent: Grapeshot Allow: / User-agent: Barkrowler Allow: / User-agent: ZoominfoBot Allow: / User-agent: Riddler Allow: / User-agent: SEOscanners.net Allow: / # City landing discovery: /cities and /cities/browse are listed in sitemap.xml # (referenced below) so crawlers can find every city hub and the browsable index. Sitemap: https://helpmefindadoctor.com/sitemap-index.xml Sitemap: https://helpmefindadoctor.com/sitemap.xml Sitemap: https://helpmefindadoctor.com/sitemap-cities.xml Sitemap: https://helpmefindadoctor.com/sitemap-specialists.xml Sitemap: https://helpmefindadoctor.com/sitemap-doctors.xml Sitemap: https://helpmefindadoctor.com/sitemap-concierge.xml # Agent / MCP discovery endpoints (allow crawling for discovery) Allow: /.well-known/api-catalog Allow: /.well-known/mcp.json Allow: /.well-known/agent-skills.json Allow: /.well-known/oauth-authorization-server Allow: /.well-known/openid-configuration Allow: /agents.json Allow: /auth.md # AI answer-engine content maps (ChatGPT, Claude, Perplexity, Gemini, Copilot). # /llms.txt is the short orientation map; /llms-full.txt enumerates every state, # city, specialty, and freshly updated profile so answer engines can cite deep # pages without crawling the whole directory. Allow: /llms.txt Allow: /llms-full.txt