{"id":48886,"date":"2026-03-15T10:03:43","date_gmt":"2026-03-15T02:03:43","guid":{"rendered":"https:\/\/www.geobok.com\/geo-toolkit\/ai-crawlability-checker\/"},"modified":"2026-04-03T10:45:06","modified_gmt":"2026-04-03T02:45:06","slug":"ai-crawlability-checker","status":"publish","type":"page","link":"https:\/\/www.geobok.com\/en\/geo-toolkit\/ai-crawlability-checker\/","title":{"rendered":"AI Crawlability Checker"},"content":{"rendered":"\n<style>\n\/* ===== C1 \u7279\u6709\u6837\u5f0f\uff1a\u722c\u866b\u72b6\u6001\u5361\u7247 ===== *\/\n.rb-summary { display: flex; align-items: center; gap: 16px; padding: 20px 24px; border-radius: var(--geo-radius-md); margin-bottom: 20px; }\n.rb-summary--ok { background: var(--geo-success-bg); border: 1px solid var(--geo-success-border); }\n.rb-summary--warning { background: var(--geo-warn-bg); border: 1px solid var(--geo-warn-border); }\n.rb-summary--blocked { background: var(--geo-danger-bg); border: 1px solid var(--geo-danger-border); }\n.rb-summary__icon { font-size: 32px; flex-shrink: 0; }\n.rb-summary__text { font-size: 15px; font-weight: 600; }\n.rb-summary--ok .rb-summary__text { color: #166534; }\n.rb-summary--warning .rb-summary__text { color: #92400e; }\n.rb-summary--blocked .rb-summary__text { color: #991b1b; }\n.rb-summary__sub { font-size: 13px; font-weight: 400; margin-top: 4px; opacity: 0.8; }\n\n\/* \u722c\u866b\u5361\u7247\u7f51\u683c *\/\n.rb-grid { display: grid; grid-template-columns: repeat(auto-fill, minmax(280px, 1fr)); gap: 12px; margin-bottom: 24px; }\n.rb-crawler { display: flex; align-items: center; gap: 14px; padding: 16px 18px; border-radius: var(--geo-radius-md); border: 1px solid var(--geo-border); background: #fff; transition: transform 0.15s var(--geo-ease), box-shadow 0.15s var(--geo-ease); }\n.rb-crawler:hover { transform: translateY(-1px); box-shadow: var(--geo-shadow-md); }\n\n.rb-light { width: 12px; height: 12px; border-radius: 50%; flex-shrink: 0; box-shadow: 0 0 6px; }\n.rb-light--allowed { background: var(--geo-success); box-shadow: 0 0 8px rgba(22,163,74,0.4); }\n.rb-light--blocked { background: var(--geo-danger); box-shadow: 0 0 8px rgba(220,38,38,0.4); }\n.rb-light--partial { background: var(--geo-warn); box-shadow: 0 0 8px rgba(217,119,6,0.4); }\n\n.rb-crawler__info { flex: 1; min-width: 0; }\n.rb-crawler__name { font-size: 14px; font-weight: 700; color: var(--geo-text-primary); display: flex; align-items: center; gap: 6px; }\n.rb-crawler__org { font-size: 11px; color: var(--geo-text-faint); background: var(--geo-bg-page); padding: 1px 6px; border-radius: 3px; font-weight: 600; }\n.rb-crawler__desc { font-size: 12px; color: var(--geo-text-muted); margin-top: 2px; }\n.rb-crawler__rule { font-size: 11px; color: var(--geo-danger); margin-top: 4px; font-family: var(--geo-font-mono); word-break: break-all; }\n\n.rb-status-tag { font-size: 12px; font-weight: 700; padding: 3px 10px; border-radius: var(--geo-radius-pill); white-space: nowrap; flex-shrink: 0; }\n.rb-status-tag--allowed { background: var(--geo-success-bg); color: #166534; }\n.rb-status-tag--blocked { background: var(--geo-danger-bg); color: #991b1b; }\n.rb-purpose-tag { font-size: 10px; font-weight: 600; padding: 1px 6px; border-radius: 3px; white-space: nowrap; }\n.rb-purpose-tag--training { background: #fef3c7; color: #92400e; }\n.rb-purpose-tag--retrieval { background: #dcfce7; color: #166534; }\n.rb-purpose-tag--both { background: #e0f2fe; color: #075985; }\n.rb-status-tag--partial { background: var(--geo-warn-bg); color: #92400e; }\n\n\/* robots.txt \u539f\u6587\u6298\u53e0\u533a *\/\n.rb-raw-toggle { display: flex; align-items: center; gap: 8px; padding: 10px 0; color: var(--geo-text-muted); font-size: 13px; font-weight: 600; cursor: pointer; border: none; background: none; font-family: var(--geo-font-sans); }\n.rb-raw-toggle:hover { color: var(--geo-primary); }\n.rb-raw-toggle__arrow { transition: transform 0.2s; display: inline-block; }\n.rb-raw-toggle__arrow--open { transform: rotate(90deg); }\n\n@media (max-width: 600px) {\n    .rb-grid { grid-template-columns: 1fr; }\n    .rb-summary { flex-direction: column; text-align: center; }\n}\n<\/style>\n\n<div class=\"geo-card\">\n    <div class=\"geo-hero\">\n        <h2 class=\"geo-hero__title\">\ud83e\udd16 AI Crawlability Check<\/h2>\n        <p class=\"geo-hero__desc\">Check if your site allows major AI crawlers. Blocked crawlers mean your content will not be indexed by AI platforms.<\/p>\n    \n<div class=\"geo-about\">\n    <details class=\"geo-about__section\">\n        <summary>\ud83d\udcd6 What can this tool do?<\/summary>\n        <div class=\"geo-about__body\">\n            <p>If AI crawlers can't reach your pages, nothing else you do matters. This tool checks your robots.txt against 10 major AI crawlers. One misconfigured rule can silently undo months of optimization.<\/p>\n            <p class=\"geo-about__ref\">See <em>Make AI Speak for You: The Definitive Guide to GEO<\/em>, Ch. 4.6<\/p>\n        <\/div>\n    <\/details>\n    <details class=\"geo-about__section\">\n        <summary>\u2753 FAQ: GEO Impact<\/summary>\n        <div class=\"geo-about__body\">\n            <div class=\"geo-faq-item\"><h4>What happens if I block AI crawlers?<\/h4><p>Your content never enters AI's retrieval index. It's not deprioritized \u2014 it doesn't exist from AI's perspective.<\/p><\/div>\n            <div class=\"geo-faq-item\"><h4>Does allowing access guarantee citations?<\/h4><p>No \u2014 crawlability is just the foundation. Your content still needs strong info density and semantic coherence.<\/p><\/div>\n            <div class=\"geo-faq-item\"><h4>Can I allow retrieval but block training?<\/h4><p>Yes. OpenAI uses GPTBot for training and OAI-SearchBot for live search. Block the former, allow the latter.<\/p><\/div>\n        <\/div>\n    <\/details>\n<\/div>\n<script type=\"application\/ld+json\">\n{\n  \"@context\": \"https:\/\/schema.org\",\n  \"@type\": \"FAQPage\",\n  \"mainEntity\": [\n    {\n      \"@type\": \"Question\",\n      \"name\": \"What happens if I block AI crawlers?\",\n      \"acceptedAnswer\": {\n        \"@type\": \"Answer\",\n        \"text\": \"Your content never enters AI's retrieval index. It's not deprioritized \u2014 it doesn't exist from AI's perspective.\"\n      }\n    },\n    {\n      \"@type\": \"Question\",\n      \"name\": \"Does allowing access guarantee citations?\",\n      \"acceptedAnswer\": {\n        \"@type\": \"Answer\",\n        \"text\": \"No \u2014 crawlability is just the foundation. Your content still needs strong info density and semantic coherence.\"\n      }\n    },\n    {\n      \"@type\": \"Question\",\n      \"name\": \"Can I allow retrieval but block training?\",\n      \"acceptedAnswer\": {\n        \"@type\": \"Answer\",\n        \"text\": \"Yes. OpenAI uses GPTBot for training and OAI-SearchBot for live search. Block the former, allow the latter.\"\n      }\n    }\n  ]\n}\n<\/script>\n\n<\/div>\n\n    <div class=\"geo-action\">\n        <div class=\"geo-input-row\">\n            <input type=\"text\" id=\"rb-domain\" class=\"geo-input\" placeholder=\"Enter domain, e.g. www.example.com\">\n            <button id=\"rb-btn\" class=\"geo-btn\">Start check<\/button>\n        <\/div>\n    <\/div>\n\n    <div id=\"rb-result\" class=\"geo-result\" style=\"padding: 0 var(--geo-space-xl) var(--geo-space-xl);\"><\/div>\n<\/div>\n\n<script>\ndocument.addEventListener('DOMContentLoaded', function(){\n    var btn = document.getElementById('rb-btn');\n    var result = document.getElementById('rb-result');\n\n    btn.addEventListener('click', async function(){\n        var domain = document.getElementById('rb-domain').value.trim();\n        if(!domain){ GeoAPI.showError(result, GeoAPI.t('err.empty_domain')); return; }\n\n        GeoAPI.disableBtn(btn, GeoAPI.t('btn.detecting'));\n        GeoAPI.showLoading(result, GeoAPI.t('robots.loading'));\n\n        try {\n            var data = await GeoAPI.post('\/geo\/robots-check', { domain: domain });\n            renderResult(data);\n        } catch(e) {\n            if(!e._geoAuthHandled) GeoAPI.showError(result, e.message);\n        } finally {\n            GeoAPI.enableBtn(btn);\n        }\n    });\n\n    function renderResult(data) {\n        var html = '';\n\n        \/* ---- 1. fetch \u5f02\u5e38\u63d0\u793a ---- *\/\n        if(data.fetch_status === 'error') {\n            GeoAPI.showError(result, '\u65e0\u6cd5\u83b7\u53d6 robots.txt\uff1a' + GeoAPI.escapeHtml(data.fetch_note));\n            return;\n        }\n\n        if(data.fetch_status === 'warning') {\n            html += '<div class=\"geo-error\" style=\"background:var(--geo-warn-bg);border-color:var(--geo-warn-border);color:#92400e;\">\u26a0 '\n                  + GeoAPI.escapeHtml(data.fetch_note) + '<\/div>';\n        }\n\n        \/* ---- 2. \u603b\u4f53\u72b6\u6001\u6458\u8981 ---- *\/\n        var isEn0 = GeoAPI.LANG === 'en';\n        var sumClass = data.overall_status === 'ok' ? 'ok' : data.overall_status === 'blocked' ? 'blocked' : 'warning';\n        var sumIcon = data.overall_status === 'ok' ? '\u2705' : data.overall_status === 'blocked' ? '\ud83d\udeab' : '\u26a0\ufe0f';\n        var subText = data.fetch_status === 'not_found'\n            ? (isEn0 ? 'No robots.txt found \u2014 all crawlers allowed by default' : '\u8be5\u57df\u540d\u672a\u8bbe\u7f6e robots.txt\uff0c\u6240\u6709\u722c\u866b\u9ed8\u8ba4\u5141\u8bb8\u8bbf\u95ee')\n            : data.allowed_count + '\/' + data.total_count + (isEn0 ? ' allowed \u00b7 ' : ' \u5141\u8bb8 \u00b7 ') + data.blocked_count + '\/' + data.total_count + (isEn0 ? ' blocked' : ' \u5c01\u9501');\n\n        html += '<div class=\"rb-summary rb-summary--' + sumClass + '\">'\n              + '<div class=\"rb-summary__icon\">' + sumIcon + '<\/div>'\n              + '<div><div class=\"rb-summary__text\">' + GeoAPI.escapeHtml(data.overall_msg) + '<\/div>'\n              + '<div class=\"rb-summary__sub\">' + GeoAPI.escapeHtml(subText) + '<\/div><\/div><\/div>';\n\n        \/* ---- 3. \u722c\u866b\u5361\u7247\u7f51\u683c ---- *\/\n        var isEn = GeoAPI.LANG === 'en';\n        html += '<div class=\"geo-section\"><h3 class=\"geo-section__title\">' + (isEn ? '\ud83d\udd0d Detailed Results' : '\ud83d\udd0d \u9010\u9879\u68c0\u6d4b\u7ed3\u679c') + '<\/h3>'\n              + '<span class=\"geo-section__desc\">' + (isEn ? 'Green = allowed \u00b7 Red = blocked \u00b7 Yellow = partial' : '\u7eff\u706f=\u5141\u8bb8 \u00b7 \u7ea2\u706f=\u5c01\u9501 \u00b7 \u9ec4\u706f=\u90e8\u5206\u9650\u5236') + '<\/span><\/div>';\n\n        \/* \u68c0\u7d22\u722c\u866b\u88ab\u5c01\u9501\u7684\u8b66\u544a *\/\n        if(data.retrieval_blocked_count > 0) {\n            html += '<div class=\"geo-error\" style=\"margin-bottom:16px;\">'\n                  + (isEn ? '\u26a0 ' + data.retrieval_blocked_count + ' retrieval\/search crawler(s) blocked \u2014 your content cannot be found by AI search features even if you allow training crawlers.'\n                         : '\u26a0 ' + data.retrieval_blocked_count + ' \u4e2a\u68c0\u7d22\u7c7b\u722c\u866b\u88ab\u5c01\u9501\u2014\u2014\u5373\u4f7f\u5141\u8bb8\u8bad\u7ec3\u722c\u866b\uff0cAI \u641c\u7d22\u529f\u80fd\u4e5f\u65e0\u6cd5\u627e\u5230\u4f60\u7684\u5185\u5bb9\u3002')\n                  + '<\/div>';\n        }\n\n        html += '<div class=\"rb-grid\">';\n        data.crawlers.forEach(function(c) {\n            var lightClass = c.status === 'allowed' ? 'allowed' : c.status === 'blocked' ? 'blocked' : 'partial';\n            var tagText = isEn\n                ? (c.status === 'allowed' ? 'Allowed' : c.status === 'blocked' ? 'Blocked' : 'Partial')\n                : (c.status === 'allowed' ? '\u5141\u8bb8' : c.status === 'blocked' ? '\u5c01\u9501' : '\u90e8\u5206');\n            var purposeCls = c.purpose === 'training' ? 'training' : c.purpose === 'retrieval' ? 'retrieval' : 'both';\n            var purposeText = isEn\n                ? (c.purpose_en || c.purpose)\n                : (c.purpose === 'training' ? '\u8bad\u7ec3' : c.purpose === 'retrieval' ? '\u68c0\u7d22' : '\u8bad\u7ec3+\u68c0\u7d22');\n\n            html += '<div class=\"rb-crawler\">'\n                  + '<div class=\"rb-light rb-light--' + lightClass + '\"><\/div>'\n                  + '<div class=\"rb-crawler__info\">'\n                  + '<div class=\"rb-crawler__name\">' + GeoAPI.escapeHtml(c.name) + ' <span class=\"rb-crawler__org\">' + GeoAPI.escapeHtml(c.org) + '<\/span>'\n                  + ' <span class=\"rb-purpose-tag rb-purpose-tag--' + purposeCls + '\">' + purposeText + '<\/span><\/div>'\n                  + '<div class=\"rb-crawler__desc\">' + GeoAPI.escapeHtml(isEn ? (c.purpose_en + ' \u2014 ' + c.org) : c.desc) + '<\/div>';\n            if(c.matched_rule) {\n                html += '<div class=\"rb-crawler__rule\">' + GeoAPI.escapeHtml(c.matched_rule) + '<\/div>';\n            }\n            html += '<\/div>'\n                  + '<span class=\"rb-status-tag rb-status-tag--' + lightClass + '\">' + tagText + '<\/span>'\n                  + '<\/div>';\n        });\n        html += '<\/div>';\n\n        \/* ---- 4. \u4fee\u590d\u5efa\u8bae ---- *\/\n        if(data.suggested_config) {\n            html += '<div class=\"geo-section\"><h3 class=\"geo-section__title\">' + (isEn ? '\ud83d\udee0\ufe0f Recommended Fix' : '\ud83d\udee0\ufe0f \u4fee\u590d\u5efa\u8bae') + '<\/h3>'\n                  + '<span class=\"geo-section__desc\">' + (isEn ? 'Add the following to your robots.txt file' : '\u5c06\u4ee5\u4e0b\u5185\u5bb9\u6dfb\u52a0\u5230\u60a8\u7684 robots.txt \u6587\u4ef6\u4e2d') + '<\/span><\/div>';\n            html += '<div class=\"geo-code-block\"><button class=\"geo-code-copy\" onclick=\"GeoAPI.copyText(document.getElementById(\\'rb-fix-code\\').textContent, this)\">' + (isEn ? 'Copy' : '\u590d\u5236') + '<\/button>'\n                  + '<code id=\"rb-fix-code\">' + GeoAPI.escapeHtml(data.suggested_config) + '<\/code><\/div>';\n        }\n\n        \/* ---- 5. robots.txt \u539f\u6587 ---- *\/\n        if(data.robots_content) {\n            html += '<button class=\"rb-raw-toggle\" onclick=\"var el=document.getElementById(\\'rb-raw-content\\');var arrow=this.querySelector(\\'.rb-raw-toggle__arrow\\');if(el.style.display===\\'none\\'){el.style.display=\\'block\\';arrow.classList.add(\\'rb-raw-toggle__arrow--open\\');}else{el.style.display=\\'none\\';arrow.classList.remove(\\'rb-raw-toggle__arrow--open\\');}\">'\n                  + '<span class=\"rb-raw-toggle__arrow\">\u25b6<\/span> ' + (isEn ? 'View robots.txt source' : '\u67e5\u770b robots.txt \u539f\u6587') + '<\/button>';\n            html += '<div id=\"rb-raw-content\" class=\"geo-code-block\" style=\"display:none;max-height:300px;overflow-y:auto;\">'\n                  + GeoAPI.escapeHtml(data.robots_content) + '<\/div>';\n        }\n\n        GeoAPI.showResult(result, html);\n    }\n});\n<\/script>\n\n        \n\n\n\n<p><\/p>\n","protected":false},"excerpt":{"rendered":"","protected":false},"author":1,"featured_media":0,"parent":48884,"menu_order":0,"comment_status":"closed","ping_status":"closed","template":"","meta":{"footnotes":""},"class_list":["post-48886","page","type-page","status-publish","hentry"],"_links":{"self":[{"href":"https:\/\/www.geobok.com\/en\/wp-json\/wp\/v2\/pages\/48886","targetHints":{"allow":["GET"]}}],"collection":[{"href":"https:\/\/www.geobok.com\/en\/wp-json\/wp\/v2\/pages"}],"about":[{"href":"https:\/\/www.geobok.com\/en\/wp-json\/wp\/v2\/types\/page"}],"author":[{"embeddable":true,"href":"https:\/\/www.geobok.com\/en\/wp-json\/wp\/v2\/users\/1"}],"replies":[{"embeddable":true,"href":"https:\/\/www.geobok.com\/en\/wp-json\/wp\/v2\/comments?post=48886"}],"version-history":[{"count":0,"href":"https:\/\/www.geobok.com\/en\/wp-json\/wp\/v2\/pages\/48886\/revisions"}],"up":[{"embeddable":true,"href":"https:\/\/www.geobok.com\/en\/wp-json\/wp\/v2\/pages\/48884"}],"wp:attachment":[{"href":"https:\/\/www.geobok.com\/en\/wp-json\/wp\/v2\/media?parent=48886"}],"curies":[{"name":"wp","href":"https:\/\/api.w.org\/{rel}","templated":true}]}}