{"id":340,"date":"2017-05-17T21:56:41","date_gmt":"2017-05-17T13:56:41","guid":{"rendered":"http:\/\/4563.org\/?p=340"},"modified":"2018-02-21T14:39:54","modified_gmt":"2018-02-21T06:39:54","slug":"robot-txt%e7%9a%84%e8%a6%8f%e5%89%87%e5%8f%8a%e5%af%ab%e6%b3%95","status":"publish","type":"post","link":"http:\/\/4563.org\/?p=340","title":{"rendered":"robot.txt\u7684\u898f\u5247\u53ca\u5beb\u6cd5"},"content":{"rendered":"

\u00a0<\/span><\/p>\n

\u5efa\u7ad9\u60f3\u6709\u641c\u7d22\u5f15\u64ce\u6d41\u91cf , \u9996\u5148\u8981\u5efa\u7acbsitemap , \u4e4b\u5f8c\u5c31\u662frobot.txt\u4e86<\/span><\/p>\n

\u56e0\u70barobot.txt\u9019\u6587\u6a94\u53ef\u4ee5\u544a\u8a34bot\u4f60\u7684\u7db2\u7ad9sitemap\u9023\u7d50 , bot\u76f4\u63a5\u53bb\u722c , \u7701\u6642\u7701\u529b<\/span><\/p>\n

\u9664\u6b64\u4e4b\u5916 , \u66f4\u53ef\u4ee5\u4f7f\u7528robot.txt\u7981\u6b62\u8a2a\u5ba2IP \u6216 \u7981\u6b62\u8a2a\u554f\u67d0\u500b\u76ee\u9304\u7b49\u7b49…<\/span><\/p>\n

=======================================================================<\/span><\/p>\n

robot.txt\u6587\u4ef6\u7684\u8a9e\u6cd5\u4e3b\u8981\u5305\u62ec\u4e86User-Agent\u3001Disallow\u3001Allow\u9019\u4e09\u500b\u547d\u4ee4\u3002<\/span><\/p>\n

1\u3001User-agent:<\/span>
\n \u5176\u5f8c\u9762\u63a5\u7684\u662f\u5177\u9ad4\u7684\u641c\u7d22\u5f15\u64ce\u7684\u8718\u86db\u7684\u540d\u5b57\u3002\u5982\u679c\u5176\u503c\u8a2d\u7f6e\u70ba\u300c*\u300d\uff0c\u5247\u8868\u793a\u5c0d\u4efb\u4f55\u8718\u86db\u90fd\u6709\u6548\u3002<\/span>
\n \u5728”robots.txt”\u6587\u4ef6\u4e2d\uff0c\u53ef\u4ee5\u6709\u591a\u689dUser-agent\u8a18\u9304\uff0c\u4f46\u300cUser-agent:*\u300d\u9019\u6a23\u7684\u8a18\u9304\u53ea\u80fd\u6709\u4e00\u689d\u3002<\/span>
\n \u5982\u679c\u5728”robots.txt”\u6587\u4ef6\u4e2d\uff0c\u52a0\u5165”User-agent: SomeSpider \u201c\u548c\u82e5\u5e72Disallow\u3001Allow\u884c\uff0c\u90a3\u9ebc\u540d\u70ba”SomeSpider”\u53ea\u53d7\u5230”User-agent: SomeSpider \u201c\u5f8c\u9762\u7684Disallow\u548cAllow\u884c\u7684\u9650\u5236\u3002<\/span>
\n \u00a0<\/span>
\n 2\u3001Disallow:<\/span>
\n \u4e0d\u5e0c\u671b\u88ab\u8a2a\u554f\u7684URL\u6216\u8005\u6587\u4ef6\u593e\uff0c\u53ef\u4ee5\u662f\u5b8c\u6574\u7684\u8def\u5f91\uff0c\u4e5f\u53ef\u4ee5\u662f\u8def\u5f91\u7684\u975e\u7a7a\u524d\u3002robots.txt\u6587\u4ef6\u4e2d\uff0c\u81f3\u5c11\u8981\u6709\u4e00\u689dDisallow\u8a18\u9304\u3002Disallow\u7684\u5167\u5bb9\u5c0d\u5927\u5c0f\u5beb\u654f\u611f\uff0c\u66f8\u5beb\u6642\u8981\u6ce8\u610f\u5927\u5c0f\u5beb\u3002<\/span>
\n \u00a0<\/span>
\n 3\u3001Allow:<\/span>
\n \u53ef\u4ee5\u88ab\u8a2a\u554f\u7684\u4e00\u7d44URL\uff0c\u8207Disallow\u9805\u76f8\u4f3c\uff0c\u9019\u500b\u503c\u53ef\u4ee5\u662f\u5b8c\u6574\u7684\u8def\u5f91\uff0c\u4e5f\u53ef\u4ee5\u662f\u8def\u5f91\u7684\u524d\u7db4\uff0cAllow\u540c\u6a23\u5c0d\u5927\u5c0f\u5beb\u654f\u611f\u3002<\/span>
\n \u00a0<\/span>
\n robot.txt\u7684\u5177\u9ad4\u5beb\u6cd5\u8209\u4f8b\u3002<\/span>
\n \u00a0<\/span>
\n \u7981\u6b62\u6240\u6709\u641c\u7d22\u5f15\u64ce\u8a2a\u554f\u7db2\u7ad9\u7684\u4efb\u4f55\u90e8\u5206<\/span>
\n User-agent: *<\/span>
\n Disallow: \/<\/span>
\n \u00a0<\/span>
\n \u5141\u8a31\u6240\u6709\u7684\u641c\u7d22\u5f15\u64ce\u8a2a\u554f<\/span>
\n User-agent: *<\/span>
\n Disallow:<\/span>
\n \u6216\u8005<\/span>
\n User-agent: *<\/span>
\n Allow: \/<\/span>
\n \u6216\u8005\u76f4\u63a5\u5efa\u4e00\u500b\u7a7a\u7684robot.txt\u6587\u4ef6\u3002<\/span>
\n \u00a0<\/span>
\n \u50c5\u7981\u6b62\u67d0\u7a2e\u8718\u86db\u8a2a\u554f\uff0c\u4f8b\u5982google<\/span>
\n User-agent: Googlebot<\/span>
\n Disallow: \/<\/span>
\n \u00a0<\/span>
\n \u50c5\u5141\u8a31\u67d0\u7a2e\u8718\u86db\u8a2a\u554f\uff0c\u4f8b\u5982google<\/span>
\n User-agent: Googlebot<\/span>
\n Disallow:<\/span>
\n \u00a0<\/span>
\n User-agent: *<\/span>
\n Disallow: \/<\/span><\/p>\n

\u7981\u6b62\u641c\u7d22\u5f15\u64ce\u8a2a\u554f\u7db2\u7ad9\u7684\u7279\u5b9a\u76ee\u9304<\/span>
\n User-agent: *<\/span>
\n Disallow: \/folder1\/<\/span>
\n Disallow: \/folder2\/<\/span>
\n \u00a0<\/span>
\n \u7981\u6b62\u8a2a\u554f\u4ee5.asp \u7d50\u5c3e\u7684\u7db2\u5740\uff1a<\/span>
\n User-Agent: *<\/span>
\n Disallow:\/*.asp$<\/span>
\n \u00a0<\/span>
\n \u7981\u6b62\u8a2a\u554f\u7db2\u7ad9\u4e2d\u6240\u6709\u7684\u52d5\u614b\u9801\u9762\uff0c\u5373\u7981\u6b62\u8a2a\u554f\u5305\u542b\u554f\u865f(?)\u7684\u7db2\u5740<\/span>
\n User-agent: *<\/span>
\n Disallow: \/*?*<\/span>
\n \u00a0<\/span>
\n \u7981\u6b62\u641c\u7d22\u5f15\u64ce\u6293\u53d6\u7db2\u7ad9\u4e0a\u6240\u6709\u7684jpg\u5716\u7247<\/span>
\n User-agent: *<\/span>
\n Disallow: \/*.jpg$<\/span>
\n \u00a0<\/span>
\n \u7981\u6b62\u6293\u53d6\u4e0d\u5b58\u5728\u7684\u7db2\u9801\u3002\u5982\u679c\u4f60\u7684\u57df\u540d\u4e4b\u524d\u66fe\u7d93\u5efa\u904e\u7ad9\uff0c\u90a3\u9ebc\u4e00\u822c\u90fd\u6703\u5b58\u5728\u4e00\u4e9b\u4f60\u7db2\u7ad9\u4e0d\u5b58\u5728\u7684\u7db2\u5740\u3002\u9019\u88e1\u5efa\u8b70\u5728\u5efa\u7ad9\u4e4b\u524d\u50cf\u641c\u7d22\u5f15\u64ce\u63d0\u4ea4\u522a\u9664\u6b64\u57df\u540d\u4e0b\u7684\u6240\u6709\u7d22\u5f15\u3002<\/span>
\n \u00a0<\/span>
\n \u63d0\u4f9bsitemap\u4f4d\u7f6e<\/span>
\n User-agent: *<\/span>
\n sitemap: http:\/\/www.domain.com\/sitemap.xml<\/span><\/p>\n","protected":false},"excerpt":{"rendered":"

\u00a0 \u5efa\u7ad9\u60f3\u6709\u641c\u7d22\u5f15\u64ce\u6d41\u91cf , \u9996\u5148…<\/p>\n","protected":false},"author":1,"featured_media":0,"comment_status":"open","ping_status":"open","sticky":false,"template":"","format":"standard","meta":[],"categories":[],"tags":[],"_links":{"self":[{"href":"http:\/\/4563.org\/index.php?rest_route=\/wp\/v2\/posts\/340"}],"collection":[{"href":"http:\/\/4563.org\/index.php?rest_route=\/wp\/v2\/posts"}],"about":[{"href":"http:\/\/4563.org\/index.php?rest_route=\/wp\/v2\/types\/post"}],"author":[{"embeddable":true,"href":"http:\/\/4563.org\/index.php?rest_route=\/wp\/v2\/users\/1"}],"replies":[{"embeddable":true,"href":"http:\/\/4563.org\/index.php?rest_route=%2Fwp%2Fv2%2Fcomments&post=340"}],"version-history":[{"count":3,"href":"http:\/\/4563.org\/index.php?rest_route=\/wp\/v2\/posts\/340\/revisions"}],"predecessor-version":[{"id":344,"href":"http:\/\/4563.org\/index.php?rest_route=\/wp\/v2\/posts\/340\/revisions\/344"}],"wp:attachment":[{"href":"http:\/\/4563.org\/index.php?rest_route=%2Fwp%2Fv2%2Fmedia&parent=340"}],"wp:term":[{"taxonomy":"category","embeddable":true,"href":"http:\/\/4563.org\/index.php?rest_route=%2Fwp%2Fv2%2Fcategories&post=340"},{"taxonomy":"post_tag","embeddable":true,"href":"http:\/\/4563.org\/index.php?rest_route=%2Fwp%2Fv2%2Ftags&post=340"}],"curies":[{"name":"wp","href":"https:\/\/api.w.org\/{rel}","templated":true}]}}