{
  "name": "EarnCitations AI crawler directory",
  "url": "https://earncitations.com/crawlers.html",
  "updated": "2026-11-21",
  "license": "https://creativecommons.org/licenses/by/4.0/",
  "licenseNote": "CC BY 4.0: reuse it, with credit to EarnCitations and a link to https://earncitations.com/crawlers.html.",
  "note": "Each agent was checked against its operator's documentation on the updated date. purpose is search, user (fetches a page because a person asked) or training. token: a robots.txt token read by another crawler, not a crawler itself. fallback: the group the agent follows when it is not named. related: agents operators document alongside their AI crawlers that fit none of the three purposes; the audit does not score them.",
  "agents": [
    {
      "bot": "GPTBot",
      "owner": "OpenAI",
      "purpose": "training",
      "docs": "https://developers.openai.com/api/docs/bots",
      "ipRanges": [
        "https://openai.com/gptbot.json"
      ],
      "note": "Crawls content that may be used to train OpenAI's generative AI foundation models. Independent of OAI-SearchBot: blocking it doesn't remove you from ChatGPT search."
    },
    {
      "bot": "OAI-SearchBot",
      "owner": "OpenAI",
      "purpose": "search",
      "docs": "https://developers.openai.com/api/docs/bots",
      "ipRanges": [
        "https://openai.com/searchbot.json"
      ],
      "note": "Surfaces websites in search results in ChatGPT's search features."
    },
    {
      "bot": "ChatGPT-User",
      "owner": "OpenAI",
      "purpose": "user",
      "docs": "https://developers.openai.com/api/docs/bots",
      "ipRanges": [
        "https://openai.com/chatgpt-user.json"
      ],
      "note": "Visits a page when a ChatGPT or Custom GPT user's request needs it. Not used for automatic crawling or to decide what appears in search."
    },
    {
      "bot": "ClaudeBot",
      "owner": "Anthropic",
      "purpose": "training",
      "docs": "https://support.claude.com/en/articles/8896518-does-anthropic-crawl-data-from-the-web-and-how-can-site-owners-block-the-crawler",
      "note": "Collects web content that could contribute to Anthropic's model training. Blocking it excludes your future content from training datasets."
    },
    {
      "bot": "Claude-SearchBot",
      "owner": "Anthropic",
      "purpose": "search",
      "docs": "https://support.claude.com/en/articles/8896518-does-anthropic-crawl-data-from-the-web-and-how-can-site-owners-block-the-crawler",
      "note": "Navigates the web to improve search result quality for Claude users. Blocking it may reduce your visibility in Claude's answers."
    },
    {
      "bot": "Claude-User",
      "owner": "Anthropic",
      "purpose": "user",
      "docs": "https://support.claude.com/en/articles/8896518-does-anthropic-crawl-data-from-the-web-and-how-can-site-owners-block-the-crawler",
      "note": "Fetches pages when someone asks Claude a question. Blocking it stops Claude retrieving your content for those users."
    },
    {
      "bot": "PerplexityBot",
      "owner": "Perplexity",
      "purpose": "search",
      "docs": "https://docs.perplexity.ai/docs/resources/perplexity-crawlers",
      "ipRanges": [
        "https://www.perplexity.ai/perplexitybot.json"
      ],
      "note": "Surfaces and links websites in Perplexity search results."
    },
    {
      "bot": "Perplexity-User",
      "owner": "Perplexity",
      "purpose": "user",
      "docs": "https://docs.perplexity.ai/docs/resources/perplexity-crawlers",
      "ipRanges": [
        "https://www.perplexity.ai/perplexity-user.json"
      ],
      "note": "Fetches pages for user actions in Perplexity. Perplexity says it generally ignores robots.txt because a user requested the fetch."
    },
    {
      "bot": "Google-Extended",
      "owner": "Google",
      "purpose": "training",
      "token": true,
      "docs": "https://developers.google.com/crawling/docs/crawlers-fetchers/google-common-crawlers",
      "note": "A robots.txt token, not a crawler. Controls whether content Google crawls may train future Gemini models and ground answers in Gemini Apps and Vertex AI. No effect on Google Search."
    },
    {
      "bot": "Googlebot",
      "owner": "Google",
      "purpose": "search",
      "docs": "https://developers.google.com/crawling/docs/crawlers-fetchers/google-common-crawlers",
      "ipRanges": [
        "https://developers.google.com/static/search/apis/ipranges/googlebot.json"
      ],
      "note": "Google Search's main crawler."
    },
    {
      "bot": "Google-Agent",
      "owner": "Google",
      "purpose": "user",
      "docs": "https://developers.google.com/crawling/docs/crawlers-fetchers/google-user-triggered-fetchers",
      "ipRanges": [
        "https://developers.google.com/static/search/apis/ipranges/user-triggered-fetchers.json",
        "https://developers.google.com/static/search/apis/ipranges/user-triggered-fetchers-google.json"
      ],
      "note": "Used by agents hosted on Google infrastructure to navigate the web and perform actions at a user's request. Google says its user-triggered fetchers generally ignore robots.txt."
    },
    {
      "bot": "Google-GeminiNotebook",
      "owner": "Google",
      "purpose": "user",
      "docs": "https://developers.google.com/crawling/docs/crawlers-fetchers/google-user-triggered-fetchers",
      "ipRanges": [
        "https://developers.google.com/static/search/apis/ipranges/user-triggered-fetchers.json",
        "https://developers.google.com/static/search/apis/ipranges/user-triggered-fetchers-google.json"
      ],
      "note": "Fetches individual URLs that Gemini Notebook users add as sources (it replaces Google-NotebookLM, which Google lists as supported until August 2026). Google says its user-triggered fetchers generally ignore robots.txt."
    },
    {
      "bot": "Bingbot",
      "owner": "Microsoft",
      "purpose": "search",
      "docs": "https://www.bing.com/webmasters/help/which-crawlers-does-bing-use-8c184ec0",
      "ipRanges": [
        "https://www.bing.com/toolbox/bingbot.json"
      ],
      "note": "Bing's standard crawler; handles most of Bing's crawling."
    },
    {
      "bot": "Applebot",
      "owner": "Apple",
      "purpose": "search",
      "fallback": "Googlebot",
      "docs": "https://support.apple.com/en-us/119829",
      "ipRanges": [
        "https://search.developer.apple.com/applebot.json"
      ],
      "note": "Powers search in Spotlight, Siri and Safari; its data may also ground AI answers in Siri and Search. Follows your Googlebot rules if Applebot isn't named."
    },
    {
      "bot": "Applebot-Extended",
      "owner": "Apple",
      "purpose": "training",
      "token": true,
      "docs": "https://support.apple.com/en-us/119829",
      "note": "A robots.txt token, not a crawler. Disallow it to opt out of training Apple's foundation models; Applebot still crawls for search."
    },
    {
      "bot": "CCBot",
      "owner": "Common Crawl",
      "purpose": "training",
      "docs": "https://commoncrawl.org/ccbot",
      "note": "Builds Common Crawl's open web archive, which many AI training datasets draw on."
    },
    {
      "bot": "Meta-ExternalAgent",
      "owner": "Meta",
      "purpose": "training",
      "docs": "https://developers.facebook.com/documentation/sharing/webmasters/web-crawlers",
      "note": "Crawls for use cases such as training foundation AI models or improving products (Meta's wording)."
    },
    {
      "bot": "Meta-ExternalFetcher",
      "owner": "Meta",
      "purpose": "user",
      "docs": "https://developers.facebook.com/documentation/sharing/webmasters/web-crawlers",
      "note": "Fetches individual links at a user's request, including for AI agents completing tasks. Meta says it may bypass robots.txt."
    },
    {
      "bot": "Bytespider",
      "owner": "ByteDance",
      "purpose": "training",
      "docs": null,
      "note": "ByteDance's crawler. We found no official documentation of its purpose or robots.txt behaviour; classed as training based on public reporting."
    },
    {
      "bot": "Amazonbot",
      "owner": "Amazon",
      "purpose": "training",
      "docs": "https://developer.amazon.com/docs/amazonbot/amazonbot",
      "note": "Used to improve Amazon products and services; Amazon says it may be used to train Amazon AI models."
    },
    {
      "bot": "Amzn-SearchBot",
      "owner": "Amazon",
      "purpose": "search",
      "docs": "https://developer.amazon.com/docs/amazonbot/amazonbot",
      "note": "Makes content eligible for search experiences such as Alexa. Not used for generative AI training."
    },
    {
      "bot": "Amzn-User",
      "owner": "Amazon",
      "purpose": "user",
      "docs": "https://developer.amazon.com/docs/amazonbot/amazonbot",
      "note": "Fetches live information for user requests such as Alexa questions. May not follow all robots.txt directives. Not used for training."
    },
    {
      "bot": "DuckAssistBot",
      "owner": "DuckDuckGo",
      "purpose": "search",
      "docs": "https://duckduckgo.com/duckduckgo-help-pages/results/duckassistbot",
      "note": "Crawls pages in real time for DuckDuckGo's AI-assisted answers, which cite their sources."
    },
    {
      "bot": "MistralAI-Index",
      "owner": "Mistral",
      "purpose": "search",
      "docs": "https://docs.mistral.ai/robots",
      "note": "Indexes content for Mistral search, used to answer questions in Vibe. Not used for training."
    },
    {
      "bot": "MistralAI-User",
      "owner": "Mistral",
      "purpose": "user",
      "docs": "https://docs.mistral.ai/robots",
      "note": "Fetches pages for user actions in Vibe."
    },
    {
      "bot": "MistralAI-Training",
      "owner": "Mistral",
      "purpose": "training",
      "docs": "https://docs.mistral.ai/robots",
      "note": "Crawls web content to build datasets for training Mistral models."
    }
  ],
  "related": [
    {
      "bot": "OAI-AdsBot",
      "owner": "OpenAI",
      "docs": "https://developers.openai.com/api/docs/bots",
      "ipRanges": [
        "https://openai.com/adsbot.json"
      ],
      "note": "Visits landing pages submitted as ads on ChatGPT to check them against OpenAI's policies, and may use their content to decide when to show the ad. OpenAI says it only visits pages submitted as ads and its data isn't used to train foundation models."
    },
    {
      "bot": "Google-CloudVertexBot",
      "owner": "Google",
      "docs": "https://developers.google.com/crawling/docs/crawlers-fetchers/google-common-crawlers",
      "note": "Crawls a site when its owner asks for it while building Vertex AI Agents. Google says the token has no effect on Google Search or other products."
    }
  ],
  "changes": [
    {
      "date": "2026-11-22",
      "change": "Added \"Other documented agents\": OAI-AdsBot (OpenAI) and Google-CloudVertexBot (Google), found on the operators' own crawler pages. Listed for completeness; they are neither answer engines, user-triggered fetchers nor training crawlers, so the audit doesn't score them."
    },
    {
      "date": "2026-11-22",
      "change": "Added ipRanges: the IP range files OpenAI, Google, Bing, Perplexity and Apple publish for their agents, so requests can be checked against them. Other operators publish no machine-readable ranges we could find."
    },
    {
      "date": "2026-11-22",
      "change": "Added Google-Agent and Google-GeminiNotebook, documented on Google's user-triggered fetchers page (it replaces Google-NotebookLM, listed as supported until August 2026). Google says these fetchers generally ignore robots.txt."
    },
    {
      "date": "2026-11-21",
      "change": "Added Amzn-SearchBot, Amzn-User, MistralAI-Index, MistralAI-Training and Meta-ExternalFetcher after re-reading each operator's documentation."
    },
    {
      "date": "2026-11-21",
      "change": "Amazonbot moved from search to training: Amazon says content it collects may be used to train Amazon's AI models."
    },
    {
      "date": "2026-11-21",
      "change": "Google-Extended: noted that Google says it also controls grounding in Gemini Apps, not only training."
    },
    {
      "date": "2026-11-21",
      "change": "Applebot: modelled Apple's rule that it follows the Googlebot group when Applebot is not named."
    }
  ]
}
