{"id":522195,"date":"2026-09-16T15:29:45","date_gmt":"2026-09-16T22:29:45","guid":{"rendered":"https:\/\/jorgep.com\/blog\/?p=522195"},"modified":"2026-09-16T15:39:48","modified_gmt":"2026-09-16T22:39:48","slug":"ai-crawlers-are-not-all-the-same","status":"publish","type":"post","link":"https:\/\/jorgep.com\/blog\/ai-crawlers-are-not-all-the-same\/","title":{"rendered":"AI Crawlers Are Not All the Same"},"content":{"rendered":"\n<div class=\"wp-block-columns has-theme-palette-7-background-color has-background is-layout-flex wp-container-core-columns-is-layout-5dc627e1 wp-block-columns-is-layout-flex\" style=\"margin-top:0;margin-bottom:0\">\n<div class=\"wp-block-column is-layout-flow wp-block-column-is-layout-flow\" style=\"flex-basis:80%\">\n<p class=\"wp-block-paragraph\">Part of: <strong> <a href=\"https:\/\/jorgep.com\/blog\/series-ai-learnings\/\">AI Learning Series Here<\/a><\/strong><\/p>\n\n\n<style>.kadence-column395113_e6e0a6-a4 > .kt-inside-inner-col,.kadence-column395113_e6e0a6-a4 > .kt-inside-inner-col:before{border-top-left-radius:0px;border-top-right-radius:0px;border-bottom-right-radius:0px;border-bottom-left-radius:0px;}.kadence-column395113_e6e0a6-a4 > .kt-inside-inner-col{column-gap:var(--global-kb-gap-sm, 1rem);}.kadence-column395113_e6e0a6-a4 > .kt-inside-inner-col{flex-direction:column;}.kadence-column395113_e6e0a6-a4 > .kt-inside-inner-col > .aligncenter{width:100%;}.kadence-column395113_e6e0a6-a4 > .kt-inside-inner-col:before{opacity:0.3;}.kadence-column395113_e6e0a6-a4{position:relative;}@media all and (max-width: 1024px){.kadence-column395113_e6e0a6-a4 > .kt-inside-inner-col{flex-direction:column;justify-content:center;}}@media all and (max-width: 767px){.kadence-column395113_e6e0a6-a4 > .kt-inside-inner-col{flex-direction:column;justify-content:center;}}<\/style>\n<div class=\"wp-block-kadence-column kadence-column395113_e6e0a6-a4\"><div class=\"kt-inside-inner-col\"><style>.kadence-column510545_f73041-db > .kt-inside-inner-col{padding-top:var(--global-kb-spacing-xs, 1rem);padding-bottom:var(--global-kb-spacing-xs, 1rem);}.kadence-column510545_f73041-db > .kt-inside-inner-col,.kadence-column510545_f73041-db > .kt-inside-inner-col:before{border-top-left-radius:0px;border-top-right-radius:0px;border-bottom-right-radius:0px;border-bottom-left-radius:0px;}.kadence-column510545_f73041-db > .kt-inside-inner-col{column-gap:var(--global-kb-gap-sm, 1rem);}.kadence-column510545_f73041-db > .kt-inside-inner-col{flex-direction:column;}.kadence-column510545_f73041-db > .kt-inside-inner-col > .aligncenter{width:100%;}.kadence-column510545_f73041-db > .kt-inside-inner-col{background-color:var(--global-palette7, #EDF2F7);}.kadence-column510545_f73041-db:hover > .kt-inside-inner-col{background-color:var(--global-palette8, #F7FAFC);background-image:none;}.kadence-column510545_f73041-db > .kt-inside-inner-col:before{opacity:0.3;}.kadence-column510545_f73041-db{position:relative;}@media all and (max-width: 1024px){.kadence-column510545_f73041-db > .kt-inside-inner-col{flex-direction:column;justify-content:center;}}@media all and (max-width: 767px){.kadence-column510545_f73041-db > .kt-inside-inner-col{flex-direction:column;justify-content:center;}}<\/style>\n<div class=\"wp-block-kadence-column kadence-column510545_f73041-db\"><div class=\"kt-inside-inner-col\"><style>.wp-block-kadence-advancedheading.kt-adv-heading510545_c22cc7-a4, .wp-block-kadence-advancedheading.kt-adv-heading510545_c22cc7-a4[data-kb-block=\"kb-adv-heading510545_c22cc7-a4\"]{text-align:center;font-size:var(--global-kb-font-size-sm, 0.9rem);font-style:normal;}.wp-block-kadence-advancedheading.kt-adv-heading510545_c22cc7-a4 mark.kt-highlight, .wp-block-kadence-advancedheading.kt-adv-heading510545_c22cc7-a4[data-kb-block=\"kb-adv-heading510545_c22cc7-a4\"] mark.kt-highlight{font-style:normal;color:#f76a0c;-webkit-box-decoration-break:clone;box-decoration-break:clone;padding-top:0px;padding-right:0px;padding-bottom:0px;padding-left:0px;}.wp-block-kadence-advancedheading.kt-adv-heading510545_c22cc7-a4 img.kb-inline-image, .wp-block-kadence-advancedheading.kt-adv-heading510545_c22cc7-a4[data-kb-block=\"kb-adv-heading510545_c22cc7-a4\"] img.kb-inline-image{width:150px;vertical-align:baseline;}<\/style>\n<p class=\"kt-adv-heading510545_c22cc7-a4 wp-block-kadence-advancedheading\" data-kb-block=\"kb-adv-heading510545_c22cc7-a4\">Quick Links:&nbsp;<a href=\"https:\/\/jorgep.com\/blog\/resources-for-learning-ai\/\">Resources for Learning AI<\/a> | <a href=\"https:\/\/jorgep.com\/blog\/keeping-up-with-ai\/\">Keep up with AI<\/a> | <a href=\"https:\/\/jorgep.com\/blog\/list-of-ai-tools\/\" data-type=\"post\" data-id=\"402818\">List of AI Tools<\/a> | <a href=\"https:\/\/jorgep.com\/blog\/local-ai-series\/\" data-type=\"page\" data-id=\"519365\">Local AI<\/a> | <a href=\"https:\/\/jorgep.com\/blog\/tag\/ai-agents\/\" data-type=\"post_tag\" data-id=\"941\">AI Agents<\/a> |  <a href=\"https:\/\/jorgep.com\/blog\/work-beyond-tomorrow-series\/\" data-type=\"page\" data-id=\"365001\">Future of Work<\/a><\/p>\n<\/div><\/div>\n<\/div><\/div>\n\n\n<style>.kb-row-layout-id395113_97845d-28 > .kt-row-column-wrap{align-content:start;}:where(.kb-row-layout-id395113_97845d-28 > .kt-row-column-wrap) > .wp-block-kadence-column{justify-content:start;}.kb-row-layout-id395113_97845d-28 > .kt-row-column-wrap{column-gap:var(--global-kb-gap-md, 2rem);row-gap:var(--global-kb-gap-none, 0rem );padding-top:var(--global-kb-spacing-xxs, 0.5rem);padding-bottom:var(--global-kb-spacing-xxs, 0.5rem);grid-template-columns:repeat(2, minmax(0, 1fr));}.kb-row-layout-id395113_97845d-28 > .kt-row-layout-overlay{opacity:0.30;}@media all and (max-width: 1024px){.kb-row-layout-id395113_97845d-28 > .kt-row-column-wrap{grid-template-columns:repeat(2, minmax(0, 1fr));}}@media all and (max-width: 767px){.kb-row-layout-id395113_97845d-28 > .kt-row-column-wrap{grid-template-columns:minmax(0, 1fr);}}<\/style><div class=\"kb-row-layout-wrap kb-row-layout-id395113_97845d-28 alignnone wp-block-kadence-rowlayout\"><div class=\"kt-row-column-wrap kt-has-2-columns kt-row-layout-equal kt-tab-layout-inherit kt-mobile-layout-row kt-row-valign-top\">\n<style>.kadence-column395113_fb3852-97 > .kt-inside-inner-col,.kadence-column395113_fb3852-97 > .kt-inside-inner-col:before{border-top-left-radius:0px;border-top-right-radius:0px;border-bottom-right-radius:0px;border-bottom-left-radius:0px;}.kadence-column395113_fb3852-97 > .kt-inside-inner-col{column-gap:var(--global-kb-gap-sm, 1rem);}.kadence-column395113_fb3852-97 > .kt-inside-inner-col{flex-direction:column;}.kadence-column395113_fb3852-97 > .kt-inside-inner-col > .aligncenter{width:100%;}.kadence-column395113_fb3852-97 > .kt-inside-inner-col:before{opacity:0.3;}.kadence-column395113_fb3852-97{position:relative;}@media all and (max-width: 1024px){.kadence-column395113_fb3852-97 > .kt-inside-inner-col{flex-direction:column;justify-content:center;}}@media all and (max-width: 767px){.kadence-column395113_fb3852-97 > .kt-inside-inner-col{flex-direction:column;justify-content:center;}}<\/style>\n<div class=\"wp-block-kadence-column kadence-column395113_fb3852-97\"><div class=\"kt-inside-inner-col\"><style>.wp-block-kadence-advancedheading.kt-adv-heading395113_0b34c1-ff, .wp-block-kadence-advancedheading.kt-adv-heading395113_0b34c1-ff[data-kb-block=\"kb-adv-heading395113_0b34c1-ff\"]{text-align:center;font-size:var(--global-kb-font-size-sm, 0.9rem);line-height:60px;font-style:normal;background-color:#f5a511;}.wp-block-kadence-advancedheading.kt-adv-heading395113_0b34c1-ff mark.kt-highlight, .wp-block-kadence-advancedheading.kt-adv-heading395113_0b34c1-ff[data-kb-block=\"kb-adv-heading395113_0b34c1-ff\"] mark.kt-highlight{font-style:normal;color:#f76a0c;-webkit-box-decoration-break:clone;box-decoration-break:clone;padding-top:0px;padding-right:0px;padding-bottom:0px;padding-left:0px;}.wp-block-kadence-advancedheading.kt-adv-heading395113_0b34c1-ff img.kb-inline-image, .wp-block-kadence-advancedheading.kt-adv-heading395113_0b34c1-ff[data-kb-block=\"kb-adv-heading395113_0b34c1-ff\"] img.kb-inline-image{width:150px;vertical-align:baseline;}<\/style>\n<p class=\"kt-adv-heading395113_0b34c1-ff wp-block-kadence-advancedheading\" data-kb-block=\"kb-adv-heading395113_0b34c1-ff\">Subscribe to <a href=\"https:\/\/go.35s.be\/jtb\" target=\"_blank\" rel=\"noreferrer noopener\"><strong>JorgeTechBits  newsletter<\/strong><\/a><\/p>\n<\/div><\/div>\n\n\n<style>.kadence-column395113_1641f9-51 > .kt-inside-inner-col,.kadence-column395113_1641f9-51 > .kt-inside-inner-col:before{border-top-left-radius:0px;border-top-right-radius:0px;border-bottom-right-radius:0px;border-bottom-left-radius:0px;}.kadence-column395113_1641f9-51 > .kt-inside-inner-col{column-gap:var(--global-kb-gap-sm, 1rem);}.kadence-column395113_1641f9-51 > .kt-inside-inner-col{flex-direction:column;}.kadence-column395113_1641f9-51 > .kt-inside-inner-col > .aligncenter{width:100%;}.kadence-column395113_1641f9-51 > .kt-inside-inner-col:before{opacity:0.3;}.kadence-column395113_1641f9-51{position:relative;}@media all and (max-width: 1024px){.kadence-column395113_1641f9-51 > .kt-inside-inner-col{flex-direction:column;justify-content:center;}}@media all and (max-width: 767px){.kadence-column395113_1641f9-51 > .kt-inside-inner-col{flex-direction:column;justify-content:center;}}<\/style>\n<div class=\"wp-block-kadence-column kadence-column395113_1641f9-51\"><div class=\"kt-inside-inner-col\"><style>.wp-block-kadence-advancedheading.kt-adv-heading395113_63dab8-35, .wp-block-kadence-advancedheading.kt-adv-heading395113_63dab8-35[data-kb-block=\"kb-adv-heading395113_63dab8-35\"]{text-align:center;font-size:var(--global-kb-font-size-sm, 0.9rem);font-style:normal;}.wp-block-kadence-advancedheading.kt-adv-heading395113_63dab8-35 mark.kt-highlight, .wp-block-kadence-advancedheading.kt-adv-heading395113_63dab8-35[data-kb-block=\"kb-adv-heading395113_63dab8-35\"] mark.kt-highlight{font-style:normal;color:#f76a0c;-webkit-box-decoration-break:clone;box-decoration-break:clone;padding-top:0px;padding-right:0px;padding-bottom:0px;padding-left:0px;}.wp-block-kadence-advancedheading.kt-adv-heading395113_63dab8-35 img.kb-inline-image, .wp-block-kadence-advancedheading.kt-adv-heading395113_63dab8-35[data-kb-block=\"kb-adv-heading395113_63dab8-35\"] img.kb-inline-image{width:150px;vertical-align:baseline;}<\/style>\n<p class=\"kt-adv-heading395113_63dab8-35 wp-block-kadence-advancedheading\" data-kb-block=\"kb-adv-heading395113_63dab8-35\">Explore the <a href=\"https:\/\/jorgep.com\/blog\/latest-token-prices\/\" data-type=\"page\" data-id=\"521255\">Latest Token Prices<\/a><\/p>\n<\/div><\/div>\n\n<\/div><\/div><\/div>\n\n\n\n<div class=\"wp-block-column is-vertically-aligned-top is-layout-flow wp-block-column-is-layout-flow\"><div class=\"wp-block-image\">\n<figure class=\"aligncenter size-large is-resized\"><a href=\"htthttps:\/\/jorgep.com\/blog\/book-dont-just-chat-delegate\/\"><img loading=\"lazy\" decoding=\"async\" width=\"640\" height=\"1024\" src=\"https:\/\/jorgep.com\/blog\/wp-content\/uploads\/CoverBook-01-640x1024.jpg\" alt=\"\" class=\"wp-image-520234\" style=\"aspect-ratio:0.6250142320391666;width:98px;height:auto\" srcset=\"https:\/\/jorgep.com\/blog\/wp-content\/uploads\/CoverBook-01-640x1024.jpg 640w, https:\/\/jorgep.com\/blog\/wp-content\/uploads\/CoverBook-01-188x300.jpg 188w, https:\/\/jorgep.com\/blog\/wp-content\/uploads\/CoverBook-01-768x1229.jpg 768w, https:\/\/jorgep.com\/blog\/wp-content\/uploads\/CoverBook-01-960x1536.jpg 960w, https:\/\/jorgep.com\/blog\/wp-content\/uploads\/CoverBook-01-1280x2048.jpg 1280w, https:\/\/jorgep.com\/blog\/wp-content\/uploads\/CoverBook-01.jpg 1600w\" sizes=\"auto, (max-width: 640px) 100vw, 640px\" \/><\/a><figcaption class=\"wp-element-caption\"><a href=\"https:\/\/jorgep.com\/blog\/book-series-ai-dont-just-chat\/\" data-type=\"page\" data-id=\"520242\">Check out the Book Series<\/a><\/figcaption><\/figure>\n<\/div><\/div>\n<\/div>\n\n\n<style>.wp-block-kadence-advancedheading.kt-adv-heading519190_af62a4-66, .wp-block-kadence-advancedheading.kt-adv-heading519190_af62a4-66[data-kb-block=\"kb-adv-heading519190_af62a4-66\"]{font-size:var(--global-kb-font-size-sm, 0.9rem);font-style:normal;}.wp-block-kadence-advancedheading.kt-adv-heading519190_af62a4-66 mark.kt-highlight, .wp-block-kadence-advancedheading.kt-adv-heading519190_af62a4-66[data-kb-block=\"kb-adv-heading519190_af62a4-66\"] mark.kt-highlight{font-style:normal;color:#f76a0c;-webkit-box-decoration-break:clone;box-decoration-break:clone;padding-top:0px;padding-right:0px;padding-bottom:0px;padding-left:0px;}.wp-block-kadence-advancedheading.kt-adv-heading519190_af62a4-66 img.kb-inline-image, .wp-block-kadence-advancedheading.kt-adv-heading519190_af62a4-66[data-kb-block=\"kb-adv-heading519190_af62a4-66\"] img.kb-inline-image{width:150px;vertical-align:baseline;}<\/style>\n<p class=\"kt-adv-heading519190_af62a4-66 wp-block-kadence-advancedheading\" data-kb-block=\"kb-adv-heading519190_af62a4-66\"><strong>Disclaimer:<\/strong> <strong>I create this content entirely on my own time, and the views expressed here are mine alone (not my employer&#8217;s)<\/strong>. Because I love leveraging new tech, I use AI tools like Gemini, ChatGPT, Claude, Perplexity and others as a &#8220;digital team&#8221; to help research and polish these articles so I can share the best possible insights with you!<\/p>\n\n\n\n<p class=\"wp-block-paragraph\">For much of the web&#8217;s history, automated access was relatively easy to understand.<\/p>\n\n\n\n<p class=\"wp-block-paragraph\">Search engines crawled websites, built indexes, and helped people find the original pages. There were exceptions, of course, but that was the basic bargain: a website made information available, a search engine made it discoverable, and some of those searches resulted in people visiting the site.<\/p>\n\n\n\n<p class=\"wp-block-paragraph\">AI is making that bargain much less straightforward.<\/p>\n\n\n\n<p class=\"wp-block-paragraph\">A crawler indexing one of my articles so someone can find it through search is doing something very different from a crawler collecting thousands of articles to train or fine-tune an AI model.<\/p>\n\n\n\n<p class=\"wp-block-paragraph\">And both are different from an AI agent visiting a website in real time because someone asked it to find information, compare options, fill out a form, or perform a task.<\/p>\n\n\n\n<p class=\"wp-block-paragraph\">Yet, for a while, website owners were essentially faced with a very blunt choice:<\/p>\n\n\n\n<p class=\"wp-block-paragraph\"><strong>Let the bots in, or block them.<\/strong><\/p>\n\n\n\n<p class=\"wp-block-paragraph\">That choice no longer matches what is actually happening on the web.<\/p>\n\n\n\n<p class=\"wp-block-paragraph\">Cloudflare recently changed its AI crawler controls, and I think the interesting part isn&#8217;t really the new dashboard settings.<\/p>\n\n\n\n<p class=\"wp-block-paragraph\">It&#8217;s the distinction Cloudflare is starting to make between <strong>search, AI training, and AI agents<\/strong>.<\/p>\n\n\n\n<p class=\"wp-block-paragraph\">Those are three very different uses of a website.<\/p>\n\n\n\n<h2 class=\"wp-block-heading\">From \u201cAI Bots\u201d to What the Bot Is Actually Doing<\/h2>\n\n\n\n<p class=\"wp-block-paragraph\">Cloudflare now organizes AI-related automated traffic around three broad behaviors:<\/p>\n\n\n\n<ul class=\"wp-block-list\">\n<li><strong>Search<\/strong> \u2014 crawling content to build an index and provide search results later.<\/li>\n\n\n\n<li><strong>Training<\/strong> \u2014 collecting content to train or fine-tune AI models.<\/li>\n\n\n\n<li><strong>Agent<\/strong> \u2014 a system visiting a website in real time on behalf of a person, such as a chat-fetch bot or browser-using agent.<\/li>\n<\/ul>\n\n\n\n<p class=\"wp-block-paragraph\">At first glance, this may look like a fairly small technical distinction.<\/p>\n\n\n\n<p class=\"wp-block-paragraph\">I don&#8217;t think it is.<\/p>\n\n\n\n<p class=\"wp-block-paragraph\">Instead of asking:<\/p>\n\n\n\n<p class=\"wp-block-paragraph\"><strong>Do I allow AI bots?<\/strong><\/p>\n\n\n\n<p class=\"wp-block-paragraph\">The better question becomes:<\/p>\n\n\n\n<p class=\"wp-block-paragraph\"><strong>What am I allowing an automated system to do with my content?<\/strong><\/p>\n\n\n\n<p class=\"wp-block-paragraph\">Search can help people discover my site.<\/p>\n\n\n\n<p class=\"wp-block-paragraph\">Training can incorporate material from my site into a model, potentially without anyone visiting the original page when that knowledge is later used.<\/p>\n\n\n\n<p class=\"wp-block-paragraph\">An agent might access the site in real time to answer someone&#8217;s question or actually do something for them.<\/p>\n\n\n\n<p class=\"wp-block-paragraph\">They may all generate automated requests to the same website.<\/p>\n\n\n\n<p class=\"wp-block-paragraph\">But they are not the same use.<\/p>\n\n\n\n<h2 class=\"wp-block-heading\">The Mixed-Use Crawler Problem<\/h2>\n\n\n\n<p class=\"wp-block-paragraph\">This gets particularly complicated with what Cloudflare calls <strong>mixed-use crawlers<\/strong>.<\/p>\n\n\n\n<p class=\"wp-block-paragraph\">Companies such as Google, Microsoft, and Apple operate traditional search crawlers while also offering AI products.<\/p>\n\n\n\n<p class=\"wp-block-paragraph\">That creates an interesting problem for website owners.<\/p>\n\n\n\n<p class=\"wp-block-paragraph\">If I block a crawler because I don&#8217;t want my articles used for AI training, could I also make those articles harder to find through normal search?<\/p>\n\n\n\n<p class=\"wp-block-paragraph\">For a blogger, publisher, small business, or really anyone who depends on organic search, that&#8217;s not a particularly attractive tradeoff.<\/p>\n\n\n\n<p class=\"wp-block-paragraph\">I may not want my articles used to train a model.<\/p>\n\n\n\n<p class=\"wp-block-paragraph\">But I certainly don&#8217;t want my site disappearing from search.<\/p>\n\n\n\n<p class=\"wp-block-paragraph\">Cloudflare&#8217;s new <strong>Disallow AI Training<\/strong> option is designed to create some separation between those activities.<\/p>\n\n\n\n<p class=\"wp-block-paragraph\">The idea is that accountable mixed-use crawlers can continue indexing the site for search while receiving a signal that the content should not be used for AI training.<\/p>\n\n\n\n<p class=\"wp-block-paragraph\">The implementations aren&#8217;t identical. Google and Apple already provide mechanisms intended to distinguish search indexing from certain AI uses, while Microsoft is moving in a similar direction.<\/p>\n\n\n\n<p class=\"wp-block-paragraph\">But the larger idea matters more than the individual crawler implementation:<\/p>\n\n\n\n<p class=\"wp-block-paragraph\"><strong>Allowing search should not automatically mean allowing AI training.<\/strong><\/p>\n\n\n\n<p class=\"wp-block-paragraph\">That seems like a fairly reasonable distinction.<\/p>\n\n\n\n<h2 class=\"wp-block-heading\">Disallow and Block Are Not the Same Thing<\/h2>\n\n\n\n<p class=\"wp-block-paragraph\">There is an important technical detail here.<\/p>\n\n\n\n<p class=\"wp-block-paragraph\">A traditional <code>robots.txt<\/code> file isn&#8217;t a security barrier.<\/p>\n\n\n\n<p class=\"wp-block-paragraph\">It publishes instructions telling crawlers what a website owner permits or doesn&#8217;t permit. Well-behaved crawlers follow those instructions.<\/p>\n\n\n\n<p class=\"wp-block-paragraph\">A crawler that chooses to ignore them still can.<\/p>\n\n\n\n<p class=\"wp-block-paragraph\">That&#8217;s why Cloudflare&#8217;s approach is interesting. It combines two ideas.<\/p>\n\n\n\n<p class=\"wp-block-paragraph\">The first is <strong>publishing the website owner&#8217;s preference<\/strong> for cooperating crawlers.<\/p>\n\n\n\n<p class=\"wp-block-paragraph\">The second is <strong>enforcing that preference at Cloudflare&#8217;s network edge<\/strong> when the traffic can be identified and blocked.<\/p>\n\n\n\n<p class=\"wp-block-paragraph\">So <strong>Disallow AI Training<\/strong> isn&#8217;t simply another name for <strong>Block<\/strong>.<\/p>\n\n\n\n<p class=\"wp-block-paragraph\">For mixed-use crawlers, the goal is to preserve legitimate search indexing while communicating that the content shouldn&#8217;t be used for training.<\/p>\n\n\n\n<p class=\"wp-block-paragraph\">For training-specific crawlers where that search tradeoff doesn&#8217;t exist, Cloudflare can block access.<\/p>\n\n\n\n<p class=\"wp-block-paragraph\">As AI use of the web expands, I think website owners are going to need both sides of this equation: a way to clearly state what automated systems are allowed to do and a practical way to enforce those choices where possible.<\/p>\n\n\n\n<h2 class=\"wp-block-heading\">Then There Are the Agents<\/h2>\n\n\n\n<p class=\"wp-block-paragraph\">This may be the part I find most interesting.<\/p>\n\n\n\n<p class=\"wp-block-paragraph\">Historically, websites have really been designed for two primary audiences:<\/p>\n\n\n\n<p class=\"wp-block-paragraph\"><strong>People and search engines.<\/strong><\/p>\n\n\n\n<p class=\"wp-block-paragraph\">We now have a third audience emerging:<\/p>\n\n\n\n<p class=\"wp-block-paragraph\"><strong>AI agents acting for people.<\/strong><\/p>\n\n\n\n<p class=\"wp-block-paragraph\">Imagine asking an AI assistant:<\/p>\n\n\n\n<blockquote class=\"wp-block-quote is-layout-flow wp-block-quote-is-layout-flow\">\n<p class=\"wp-block-paragraph\">Find three restaurants nearby with outdoor seating, compare their menus, and tell me which ones have live music tonight.<\/p>\n<\/blockquote>\n\n\n\n<p class=\"wp-block-paragraph\">The agent might visit several websites, read event calendars, compare listings, and return an answer.<\/p>\n\n\n\n<p class=\"wp-block-paragraph\">You may never visit any of those websites yourself.<\/p>\n\n\n\n<p class=\"wp-block-paragraph\">Or imagine an agent reading support documentation, checking product availability, filling out a scheduling form, comparing products, or interacting with a customer portal.<\/p>\n\n\n\n<p class=\"wp-block-paragraph\">That&#8217;s not traditional search.<\/p>\n\n\n\n<p class=\"wp-block-paragraph\">And it&#8217;s certainly not AI training.<\/p>\n\n\n\n<p class=\"wp-block-paragraph\">It&#8217;s <strong>delegated interaction<\/strong>.<\/p>\n\n\n\n<p class=\"wp-block-paragraph\">A person has given software permission to interact with websites on their behalf.<\/p>\n\n\n\n<p class=\"wp-block-paragraph\">That creates an entirely different set of questions.<\/p>\n\n\n\n<p class=\"wp-block-paragraph\">Reading a public article is one thing.<\/p>\n\n\n\n<p class=\"wp-block-paragraph\">Completing a transaction is another.<\/p>\n\n\n\n<p class=\"wp-block-paragraph\">Accessing a customer account, making an appointment, submitting a form, or purchasing something brings authorization, identity, security, fraud prevention, and accountability into the picture.<\/p>\n\n\n\n<p class=\"wp-block-paragraph\">Crawler controls are only the beginning.<\/p>\n\n\n\n<p class=\"wp-block-paragraph\">The agentic web is eventually going to need better ways for websites and agents to understand who is acting, who authorized the action, what the agent is allowed to do, and where the boundaries are.<\/p>\n\n\n\n<h2 class=\"wp-block-heading\">We Need a Better Vocabulary<\/h2>\n\n\n\n<p class=\"wp-block-paragraph\">Another thing I like about Cloudflare&#8217;s approach is that it starts giving us better language for talking about how content is used.<\/p>\n\n\n\n<p class=\"wp-block-paragraph\">Cloudflare&#8217;s Content Signals framework distinguishes between:<\/p>\n\n\n\n<ul class=\"wp-block-list\">\n<li><code>search<\/code> \u2014 building search indexes and providing search results.<\/li>\n\n\n\n<li><code>ai-input<\/code> \u2014 using content as input to an AI system when answering a request.<\/li>\n\n\n\n<li><code>ai-train<\/code> \u2014 using content to train or fine-tune a model.<\/li>\n<\/ul>\n\n\n\n<p class=\"wp-block-paragraph\">That middle category is particularly interesting.<\/p>\n\n\n\n<p class=\"wp-block-paragraph\">Suppose I publish an article explaining how something works.<\/p>\n\n\n\n<p class=\"wp-block-paragraph\">There is a meaningful difference between an AI system reading that article today because someone asked a question and an AI company collecting the article so it can become part of a future model.<\/p>\n\n\n\n<p class=\"wp-block-paragraph\">One is immediate retrieval.<\/p>\n\n\n\n<p class=\"wp-block-paragraph\">The other is model development.<\/p>\n\n\n\n<p class=\"wp-block-paragraph\">And there are more distinctions coming.<\/p>\n\n\n\n<p class=\"wp-block-paragraph\">Can an AI system quote the material?<\/p>\n\n\n\n<p class=\"wp-block-paragraph\">Can it summarize it?<\/p>\n\n\n\n<p class=\"wp-block-paragraph\">Can it store it?<\/p>\n\n\n\n<p class=\"wp-block-paragraph\">Should it provide attribution?<\/p>\n\n\n\n<p class=\"wp-block-paragraph\">Should it link back?<\/p>\n\n\n\n<p class=\"wp-block-paragraph\">Can an agent act on information from the site?<\/p>\n\n\n\n<p class=\"wp-block-paragraph\">These questions were much less important when the primary automated visitor to a website was a search crawler building an index.<\/p>\n\n\n\n<p class=\"wp-block-paragraph\">They become much more important when AI systems can consume information, synthesize it, and act on it.<\/p>\n\n\n\n<h2 class=\"wp-block-heading\">Why Website Owners Should Pay Attention<\/h2>\n\n\n\n<p class=\"wp-block-paragraph\">Cloudflare is replacing its older broad <strong>Block AI Bots<\/strong> approach with more granular controls for Search, Training, and Agents.<\/p>\n\n\n\n<p class=\"wp-block-paragraph\">For many Cloudflare customers, the migration will happen automatically based on their previous settings.<\/p>\n\n\n\n<p class=\"wp-block-paragraph\">So this isn&#8217;t necessarily something that requires an urgent configuration change.<\/p>\n\n\n\n<p class=\"wp-block-paragraph\">But I do think it&#8217;s worth opening the dashboard and looking at the new controls.<\/p>\n\n\n\n<p class=\"wp-block-paragraph\">Not because everyone should block AI.<\/p>\n\n\n\n<p class=\"wp-block-paragraph\">I don&#8217;t think that&#8217;s the right conclusion.<\/p>\n\n\n\n<p class=\"wp-block-paragraph\">The better question is what role you want these systems to play on your site.<\/p>\n\n\n\n<p class=\"wp-block-paragraph\">For example, a retailer may welcome AI agents that make its products easier to discover or buy.<\/p>\n\n\n\n<p class=\"wp-block-paragraph\">A publisher whose business depends on page views may feel very differently about an AI system providing complete answers without sending the reader to the original article.<\/p>\n\n\n\n<p class=\"wp-block-paragraph\">A technical blogger might be perfectly comfortable with an AI assistant retrieving an article to answer someone&#8217;s question while still preferring that the entire site not be collected for model training.<\/p>\n\n\n\n<p class=\"wp-block-paragraph\">Different sites have different economics and different reasons for publishing.<\/p>\n\n\n\n<p class=\"wp-block-paragraph\">Their AI policies should probably be different too.<\/p>\n\n\n\n<h2 class=\"wp-block-heading\">The Larger Shift<\/h2>\n\n\n\n<p class=\"wp-block-paragraph\">That&#8217;s why I think this Cloudflare change is more interesting than it initially appears.<\/p>\n\n\n\n<p class=\"wp-block-paragraph\">The old web bargain was imperfect, but understandable:<\/p>\n\n\n\n<p class=\"wp-block-paragraph\"><strong>Search engines crawled sites.<br>Search engines built indexes.<br>People searched.<br>Some of those people visited the original sites.<\/strong><\/p>\n\n\n\n<p class=\"wp-block-paragraph\">AI complicates that exchange.<\/p>\n\n\n\n<p class=\"wp-block-paragraph\">A model can consume content during training.<\/p>\n\n\n\n<p class=\"wp-block-paragraph\">An AI assistant can retrieve information from multiple sources and synthesize an answer without the person necessarily visiting those sources.<\/p>\n\n\n\n<p class=\"wp-block-paragraph\">And an AI agent can go one step further \u2014 visiting websites and actually doing things on behalf of the user.<\/p>\n\n\n\n<p class=\"wp-block-paragraph\">This doesn&#8217;t mean AI access is inherently harmful.<\/p>\n\n\n\n<p class=\"wp-block-paragraph\">It means different kinds of access create different kinds of value \u2014 and different costs \u2014 for the people who create and operate websites.<\/p>\n\n\n\n<p class=\"wp-block-paragraph\">That&#8217;s why lumping everything together under <strong>\u201cAI bots\u201d<\/strong> doesn&#8217;t make much sense anymore.<\/p>\n\n\n\n<p class=\"wp-block-paragraph\">Cloudflare hasn&#8217;t solved the larger problem.<\/p>\n\n\n\n<p class=\"wp-block-paragraph\"><code>robots.txt<\/code> and emerging content signals still depend partly on cooperation. Standards are evolving. Crawler identities can be difficult to verify. AI-generated answers introduce separate questions about attribution and traffic. And we&#8217;re only beginning to figure out the rules for agents interacting with websites on our behalf.<\/p>\n\n\n\n<p class=\"wp-block-paragraph\">But we&#8217;re starting to develop a more useful middle ground between:<\/p>\n\n\n\n<p class=\"wp-block-paragraph\"><strong>Let every automated system use everything.<\/strong><\/p>\n\n\n\n<p class=\"wp-block-paragraph\">and<\/p>\n\n\n\n<p class=\"wp-block-paragraph\"><strong>Block everything.<\/strong><\/p>\n\n\n\n<p class=\"wp-block-paragraph\">For my own sites, the principle is fairly straightforward.<\/p>\n\n\n\n<p class=\"wp-block-paragraph\">I want people to be able to <strong>find my work<\/strong>.<\/p>\n\n\n\n<p class=\"wp-block-paragraph\">I want useful AI tools to be able to <strong>help people discover and use it<\/strong>.<\/p>\n\n\n\n<p class=\"wp-block-paragraph\">But that doesn&#8217;t automatically mean every automated system should have unrestricted permission to <strong>collect it for training, reproduce it in answers, or interact with my sites however it chooses<\/strong>.<\/p>\n\n\n\n<p class=\"wp-block-paragraph\">Those are different activities.<\/p>\n\n\n\n<p class=\"wp-block-paragraph\">And as the web becomes increasingly agentic, I think understanding those differences is going to matter a lot more.<\/p>\n\n\n\n<h2 class=\"wp-block-heading\">References<\/h2>\n\n\n\n<ul class=\"wp-block-list\">\n<li><a href=\"https:\/\/blog.cloudflare.com\/accountable-mixed-use-ai-crawlers\/\">Cloudflare \u2014 Have it both ways: stay discoverable in search while disallowing AI training<\/a><\/li>\n\n\n\n<li><a href=\"https:\/\/blog.cloudflare.com\/content-independence-day-ai-options\/\">Cloudflare \u2014 Your site, your rules: new AI traffic options for all customers<\/a><\/li>\n\n\n\n<li><a href=\"https:\/\/developers.cloudflare.com\/bots\/additional-configurations\/block-ai-bots\/\">Cloudflare Docs \u2014 Block AI Bots and AI crawler policies<\/a><\/li>\n\n\n\n<li><a href=\"https:\/\/developers.cloudflare.com\/bots\/additional-configurations\/managed-robots-txt\/\">Cloudflare Docs \u2014 robots.txt and AI Content Signals<\/a><\/li>\n\n\n\n<li><a href=\"https:\/\/blog.cloudflare.com\/bot-preference-sync\/\">Cloudflare \u2014 Say it once: introducing Bot Preference Sync<\/a><\/li>\n<\/ul>\n","protected":false},"excerpt":{"rendered":"<p>Search crawlers, AI training bots, and AI agents may all visit the same website, but they are doing very different things. Cloudflare&#8217;s new controls point toward a more nuanced way of deciding how automated systems can use our content.<\/p>\n","protected":false},"author":2,"featured_media":522198,"comment_status":"closed","ping_status":"open","sticky":false,"template":"","format":"standard","meta":{"_kad_blocks_custom_css":"","_kad_blocks_head_custom_js":"","_kad_blocks_body_custom_js":"","_kad_blocks_footer_custom_js":"","ngg_post_thumbnail":0,"episode_type":"","audio_file":"","podmotor_file_id":"","podmotor_episode_id":"","cover_image":"","cover_image_id":"","duration":"","filesize":"","filesize_raw":"","date_recorded":"","explicit":"","block":"","itunes_episode_number":"","itunes_title":"","itunes_season_number":"","itunes_episode_type":"","_kad_post_transparent":"","_kad_post_title":"","_kad_post_layout":"","_kad_post_sidebar_id":"","_kad_post_content_style":"","_kad_post_vertical_padding":"","_kad_post_feature":"","_kad_post_feature_position":"","_kad_post_header":false,"_kad_post_footer":false,"_kad_post_classname":"","footnotes":""},"categories":[1068],"tags":[],"class_list":["post-522195","post","type-post","status-publish","format-standard","has-post-thumbnail","hentry","category-ai-emerging-technology"],"taxonomy_info":{"category":[{"value":1068,"label":"AI &amp; Emerging Technology"}]},"featured_image_src_large":["https:\/\/jorgep.com\/blog\/wp-content\/uploads\/FeaturedImage-AICrawlers-1024x384.jpg",1024,384,true],"author_info":{"display_name":"Jorge Pereira","author_link":"https:\/\/jorgep.com\/blog\/author\/jorge\/"},"comment_info":0,"category_info":[{"term_id":1068,"name":"AI &amp; Emerging Technology","slug":"ai-emerging-technology","term_group":0,"term_taxonomy_id":1078,"taxonomy":"category","description":"","parent":0,"count":224,"filter":"raw","cat_ID":1068,"category_count":224,"category_description":"","cat_name":"AI &amp; Emerging Technology","category_nicename":"ai-emerging-technology","category_parent":0}],"tag_info":false,"_links":{"self":[{"href":"https:\/\/jorgep.com\/blog\/wp-json\/wp\/v2\/posts\/522195","targetHints":{"allow":["GET"]}}],"collection":[{"href":"https:\/\/jorgep.com\/blog\/wp-json\/wp\/v2\/posts"}],"about":[{"href":"https:\/\/jorgep.com\/blog\/wp-json\/wp\/v2\/types\/post"}],"author":[{"embeddable":true,"href":"https:\/\/jorgep.com\/blog\/wp-json\/wp\/v2\/users\/2"}],"replies":[{"embeddable":true,"href":"https:\/\/jorgep.com\/blog\/wp-json\/wp\/v2\/comments?post=522195"}],"version-history":[{"count":2,"href":"https:\/\/jorgep.com\/blog\/wp-json\/wp\/v2\/posts\/522195\/revisions"}],"predecessor-version":[{"id":522200,"href":"https:\/\/jorgep.com\/blog\/wp-json\/wp\/v2\/posts\/522195\/revisions\/522200"}],"wp:featuredmedia":[{"embeddable":true,"href":"https:\/\/jorgep.com\/blog\/wp-json\/wp\/v2\/media\/522198"}],"wp:attachment":[{"href":"https:\/\/jorgep.com\/blog\/wp-json\/wp\/v2\/media?parent=522195"}],"wp:term":[{"taxonomy":"category","embeddable":true,"href":"https:\/\/jorgep.com\/blog\/wp-json\/wp\/v2\/categories?post=522195"},{"taxonomy":"post_tag","embeddable":true,"href":"https:\/\/jorgep.com\/blog\/wp-json\/wp\/v2\/tags?post=522195"}],"curies":[{"name":"wp","href":"https:\/\/api.w.org\/{rel}","templated":true}]}}